Version 0.1.2

This commit is contained in:
louis 2014-05-09 01:05:21 +02:00
parent 30008b3000
commit ca71c83e5d
24 changed files with 3061 additions and 2305 deletions

11
HISTORY
View File

@ -14,3 +14,14 @@ Version 0.1.0
- fixed bug that scraped events are not displayed randomly - fixed bug that scraped events are not displayed randomly
Version 0.1.1 Version 0.1.1
- update of db api
- prepare statements only once
- added statement statistic
- minor changes
- changed mail loop timing
- added path detection for libmysql
- fixed epgd state detection
- added cppcheck to Makefile
Version 0.1.2

View File

@ -25,7 +25,7 @@ TMPDIR ?= /tmp
### The compiler options: ### The compiler options:
export CFLAGS = $(call PKGCFG,cflags) export CFLAGS = $(call PKGCFG,cflags)
export CXXFLAGS = $(call PKGCFG,cxxflags) export CXXFLAGS = $(call PKGCFG,cxxflags) -Wno-unused-result -Wunused-value -Wunused-variable -Wreturn-type -Wuninitialized -Wsign-compare
### The version number of VDR's plugin API: ### The version number of VDR's plugin API:
@ -35,7 +35,7 @@ APIVERSION = $(call PKGCFG,apiversion)
-include $(PLGCFG) -include $(PLGCFG)
LIBS = -lmysqlclient_r -luuid LIBS = $(shell mysql_config --libs_r) -luuid
### The name of the distribution archive: ### The name of the distribution archive:
@ -48,9 +48,9 @@ SOFILE = libvdr-$(PLUGIN).so
### Includes and Defines (add further entries here): ### Includes and Defines (add further entries here):
INCLUDES += INCLUDES += $(shell mysql_config --include)
DEFINES += -DPLUGIN_NAME_I18N='"$(PLUGIN)"' DEFINES += -DPLUGIN_NAME_I18N='"$(PLUGIN)"' -DLOG_PREFIX='"$(PLUGIN): "'
DEFINES += -DVDR_PLUGIN -DUSEUUID DEFINES += -DVDR_PLUGIN -DUSEUUID
ifeq ($(IMAGELIB), imagemagick) ifeq ($(IMAGELIB), imagemagick)
@ -129,4 +129,7 @@ dist: $(I18Npo) clean
clean: clean:
@-rm -f $(PODIR)/*.mo $(PODIR)/*.pot @-rm -f $(PODIR)/*.mo $(PODIR)/*.pot
@-rm -f $(OBJS) $(DEPFILE) *.so *.tgz core* *~ @-rm -f $(OBJS) $(DEPFILE) *.so *.tgz core* *~ lib/*~
cppchk:
cppcheck --template="{file}:{line}:{severity}:{message}" --quiet --force *.c *.h lib/*.c lib/*.h

View File

@ -1,7 +1,14 @@
BASELIBS = -lrt -lz -lmysqlclient -lcurl -luuid
BASELIBS += $(shell mysql_config --libs)
CFLAGS += $(shell mysql_config --include)
all: all:
g++ -ggdb -DPLGDIR='"."' test.c common.c config.c db.c tabledef.c -lrt -lz -lmysqlclient -o t g++ -ggdb -DPLGDIR='"."' $(CFLAGS) test.c dbdict.c common.c config.c db.c tabledef.c $(BASELIBS) -o t
demo: demo.c
g++ -ggdb -DUSEUUID -DPLGDIR='"."' $(CFLAGS) demo.c common.c db.c tabledef.c config.c $(BASELIBS) -o demo
clean: clean:
rm -f *.o *.a *~ core rm -f *.o *.a *~ core demo

View File

@ -6,6 +6,7 @@
*/ */
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/time.h>
#ifdef USEUUID #ifdef USEUUID
# include <uuid/uuid.h> # include <uuid/uuid.h>
@ -17,6 +18,7 @@
#include <syslog.h> #include <syslog.h>
#include <unistd.h> #include <unistd.h>
#include <zlib.h> #include <zlib.h>
#include <errno.h>
#ifdef USELIBARCHIVE #ifdef USELIBARCHIVE
# include <archive.h> # include <archive.h>
@ -54,7 +56,7 @@ void tell(int eloquence, const char* format, ...)
va_start(ap, format); va_start(ap, format);
#ifdef VDR_PLUGIN #ifdef VDR_PLUGIN
snprintf(t, sizeBuffer, "scraper2vdr: "); snprintf(t, sizeBuffer, LOG_PREFIX);
#endif #endif
vsnprintf(t+strlen(t), sizeBuffer-strlen(t), format, ap); vsnprintf(t+strlen(t), sizeBuffer-strlen(t), format, ap);
@ -62,10 +64,18 @@ void tell(int eloquence, const char* format, ...)
if (EPG2VDRConfig.logstdout) if (EPG2VDRConfig.logstdout)
{ {
char buf[50+TB]; char buf[50+TB];
time_t now; timeval tp;
time(&now);
strftime(buf, 50, "%y.%m.%d %H:%M:%S", localtime(&now)); gettimeofday(&tp, 0);
tm* tm = localtime(&tp.tv_sec);
sprintf(buf,"%2.2d:%2.2d:%2.2d,%3.3ld ",
tm->tm_hour, tm->tm_min, tm->tm_sec,
tp.tv_usec / 1000);
printf("%s %s\n", buf, t); printf("%s %s\n", buf, t);
} }
else else
syslog(LOG_ERR, "%s", t); syslog(LOG_ERR, "%s", t);
@ -73,6 +83,36 @@ void tell(int eloquence, const char* format, ...)
va_end(ap); va_end(ap);
} }
//***************************************************************************
// Save Realloc
//***************************************************************************
char* srealloc(void* ptr, size_t size)
{
void* n = realloc(ptr, size);
if (!n)
{
free(ptr);
ptr = 0;
}
return (char*)n;
}
//***************************************************************************
// us now
//***************************************************************************
double usNow()
{
struct timeval tp;
gettimeofday(&tp, 0);
return tp.tv_sec * 1000000.0 + tp.tv_usec;
}
//*************************************************************************** //***************************************************************************
// Host ID // Host ID
//*************************************************************************** //***************************************************************************
@ -121,6 +161,38 @@ void toUpper(std::string& str)
free(dest); free(dest);
} }
//***************************************************************************
// To Case (UTF-8 save)
//***************************************************************************
const char* toCase(Case cs, char* str)
{
char* s = str;
int lenSrc = strlen(str);
int csSrc; // size of character
for (int ps = 0; ps < lenSrc; ps += csSrc)
{
csSrc = max(mblen(&s[ps], lenSrc-ps), 1);
if (csSrc == 1)
s[ps] = cs == cUpper ? toupper(s[ps]) : tolower(s[ps]);
else if (csSrc == 2 && s[ps] == (char)0xc3 && s[ps+1] >= (char)0xa0)
{
s[ps] = s[ps];
s[ps+1] = cs == cUpper ? toupper(s[ps+1]) : tolower(s[ps+1]);
}
else
{
for (int i = 0; i < csSrc; i++)
s[ps+i] = s[ps+i];
}
}
return str;
}
void removeChars(std::string& str, const char* ignore) void removeChars(std::string& str, const char* ignore)
{ {
const char* s = str.c_str(); const char* s = str.c_str();
@ -329,6 +401,86 @@ const char* c2s(char c, char* buf)
return buf; return buf;
} }
//***************************************************************************
// Store To File
//***************************************************************************
int storeToFile(const char* filename, const char* data, int size)
{
FILE* fout;
if ((fout = fopen(filename, "w+")))
{
fwrite(data, sizeof(char), size, fout);
fclose(fout);
}
else
{
tell(0, "Error, can't store '%s' to filesystem '%s'", filename, strerror(errno));
return fail;
}
return success;
}
//***************************************************************************
// Load From File
//***************************************************************************
int loadFromFile(const char* infile, MemoryStruct* data)
{
FILE* fin;
struct stat sb;
data->clear();
if (!fileExists(infile))
{
tell(0, "File '%s' not found'", infile);
return fail;
}
if (stat(infile, &sb) < 0)
{
tell(0, "Can't get info of '%s', error was '%s'", infile, strerror(errno));
return fail;
}
if ((fin = fopen(infile, "r")))
{
const char* sfx = suffixOf(infile);
data->size = sb.st_size;
data->modTime = sb.st_mtime;
data->memory = (char*)malloc(data->size);
fread(data->memory, sizeof(char), data->size, fin);
fclose(fin);
sprintf(data->tag, "%ld", data->size);
if (strcmp(sfx, "gz") == 0)
sprintf(data->contentEncoding, "gzip");
if (strcmp(sfx, "js") == 0)
sprintf(data->contentType, "application/javascript");
else if (strcmp(sfx, "png") == 0 || strcmp(sfx, "jpg") == 0 || strcmp(sfx, "gif") == 0)
sprintf(data->contentType, "image/%s", sfx);
else if (strcmp(sfx, "ico") == 0)
strcpy(data->contentType, "image/x-icon");
else
sprintf(data->contentType, "text/%s", sfx);
}
else
{
tell(0, "Error, can't open '%s' for reading, error was '%s'", infile, strerror(errno));
return fail;
}
return success;
}
//*************************************************************************** //***************************************************************************
// TOOLS // TOOLS
//*************************************************************************** //***************************************************************************
@ -338,6 +490,33 @@ int isEmpty(const char* str)
return !str || !*str; return !str || !*str;
} }
const char* notNull(const char* str)
{
if (!str)
return "<null>";
return str;
}
int isZero(const char* str)
{
const char* p = str;
while (p && *p)
{
if (*p != '0')
return no;
p++;
}
return yes;
}
//***************************************************************************
//
//***************************************************************************
char* sstrcpy(char* dest, const char* src, int max) char* sstrcpy(char* dest, const char* src, int max)
{ {
if (!dest || !src) if (!dest || !src)
@ -356,11 +535,21 @@ int isLink(const char* path)
if (lstat(path, &sb) == 0) if (lstat(path, &sb) == 0)
return S_ISLNK(sb.st_mode); return S_ISLNK(sb.st_mode);
tell(0, "Error: Detecting state for '%s' failed, error was '%m'", path); tell(0, "Error: Detecting state for '%s' failed, error was '%s'", path, strerror(errno));
return false; return false;
} }
const char* suffixOf(const char* path)
{
const char* p;
if (path && (p = strrchr(path, '.')))
return p+1;
return "";
}
int fileSize(const char* path) int fileSize(const char* path)
{ {
struct stat sb; struct stat sb;
@ -368,9 +557,21 @@ int fileSize(const char* path)
if (lstat(path, &sb) == 0) if (lstat(path, &sb) == 0)
return sb.st_size; return sb.st_size;
tell(0, "Error: Detecting state for '%s' failed, error was '%m'", path); tell(0, "Error: Detecting state for '%s' failed, error was '%s'", path, strerror(errno));
return false; return 0;
}
time_t fileModTime(const char* path)
{
struct stat sb;
if (lstat(path, &sb) == 0)
return sb.st_mtime;
tell(0, "Error: Detecting state for '%s' failed, error was '%s'", path, strerror(errno));
return 0;
} }
@ -390,7 +591,7 @@ int createLink(const char* link, const char* dest, int force)
if (symlink(dest, link) != 0) if (symlink(dest, link) != 0)
{ {
tell(0, "Failed to create symlink '%s', error was '%m'", link); tell(0, "Failed to create symlink '%s', error was '%s'", link, strerror(errno));
return fail; return fail;
} }
} }
@ -408,7 +609,7 @@ int removeFile(const char* filename)
if (unlink(filename) != 0) if (unlink(filename) != 0)
{ {
tell(0, "Can't remove file '%s', '%m'", filename); tell(0, "Can't remove file '%s', '%s'", filename, strerror(errno));
return 1; return 1;
} }
@ -432,7 +633,7 @@ int chkDir(const char* path)
if (mkdir(path, ACCESSPERMS) == -1) if (mkdir(path, ACCESSPERMS) == -1)
{ {
tell(0, "Can't create directory '%m'"); tell(0, "Can't create directory '%s'", strerror(errno));
return fail; return fail;
} }
} }
@ -541,6 +742,63 @@ int gunzip(MemoryStruct* zippedData, MemoryStruct* unzippedData)
return success; return success;
} }
//***************************************************************************
// gzip
//***************************************************************************
int _gzip(Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen)
{
z_stream stream;
int res;
stream.next_in = (Bytef *)source;
stream.avail_in = (uInt)sourceLen;
stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;
if ((res = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY)) != Z_OK)
return res;
res = deflate(&stream, Z_FINISH);
if (res != Z_STREAM_END)
{
deflateEnd(&stream);
return res == Z_OK ? Z_BUF_ERROR : res;
}
*destLen = stream.total_out;
res = deflateEnd(&stream);
return res;
}
int gzip(MemoryStruct* data, MemoryStruct* zippedData)
{
int res;
uLong sizeMax = compressBound(data->size) + 512;
zippedData->clear();
zippedData->memory = (char*)malloc(sizeMax);
if ((res = _gzip((Bytef*)zippedData->memory, &sizeMax, (Bytef*)data->memory, data->size)) != Z_OK)
{
tellZipError(res, " during compression", "");
return fail;
}
zippedData->copyAttributes(data);
zippedData->size = sizeMax;
sprintf(zippedData->contentEncoding, "gzip");
return success;
}
//************************************************************************* //*************************************************************************
// tellZipError // tellZipError
//************************************************************************* //*************************************************************************
@ -554,11 +812,11 @@ void tellZipError(int errorCode, const char* op, const char* msg)
{ {
case Z_OK: return; case Z_OK: return;
case Z_STREAM_END: return; case Z_STREAM_END: return;
case Z_MEM_ERROR: tell(0, "Error: Not enough memory to unzip file%s!\n", op); return; case Z_MEM_ERROR: tell(0, "Error: Not enough memory to zip/unzip file%s!\n", op); return;
case Z_BUF_ERROR: tell(0, "Error: Couldn't unzip data due to output buffer size problem%s!\n", op); return; case Z_BUF_ERROR: tell(0, "Error: Couldn't zip/unzip data due to output buffer size problem%s!\n", op); return;
case Z_DATA_ERROR: tell(0, "Error: Zipped input data corrupted%s! Details: %s\n", op, msg); return; case Z_DATA_ERROR: tell(0, "Error: Zipped input data corrupted%s! Details: %s\n", op, msg); return;
case Z_STREAM_ERROR: tell(0, "Error: Invalid stream structure%s. Details: %s\n", op, msg); return; case Z_STREAM_ERROR: tell(0, "Error: Invalid stream structure%s. Details: %s\n", op, msg); return;
default: tell(0, "Error: Couldn't unzip data for unknown reason (%6d)%s!\n", errorCode, op); return; default: tell(0, "Error: Couldn't zip/unzip data for unknown reason (%6d)%s!\n", errorCode, op); return;
} }
} }
@ -659,7 +917,7 @@ int unzip(const char* file, const char* filter, char*& buffer, int& size, char*
if (r != ARCHIVE_OK) if (r != ARCHIVE_OK)
{ {
tell(0, "Error: Open '%s' failed - %m", file); tell(0, "Error: Open '%s' failed - %s", file, strerror(errno));
return 1; return 1;
} }
@ -720,13 +978,13 @@ LogDuration::LogDuration(const char* aMessage, int aLogLevel)
LogDuration::~LogDuration() LogDuration::~LogDuration()
{ {
tell(logLevel, "duration '%s' was (%dms)", tell(logLevel, "duration '%s' was (%ldms)",
message, cTimeMs::Now() - durationStart); message, cTimeMs::Now() - durationStart);
} }
void LogDuration::show(const char* label) void LogDuration::show(const char* label)
{ {
tell(logLevel, "elapsed '%s' at '%s' was (%dms)", tell(logLevel, "elapsed '%s' at '%s' was (%ldms)",
message, label, cTimeMs::Now() - durationStart); message, label, cTimeMs::Now() - durationStart);
} }
@ -785,7 +1043,7 @@ int createMd5OfFile(const char* path, const char* name, md5* md5)
if (!(f = fopen(file, "r"))) if (!(f = fopen(file, "r")))
{ {
tell(0, "Fatal: Can't access '%s'; %m", file); tell(0, "Fatal: Can't access '%s'; %s", file, strerror(errno));
free(file); free(file);
return fail; return fail;
} }
@ -810,3 +1068,137 @@ int createMd5OfFile(const char* path, const char* name, md5* md5)
} }
#endif // USEMD5 #endif // USEMD5
//***************************************************************************
// Url Unescape
//***************************************************************************
/*
* The buffer pointed to by @dst must be at least strlen(@src) bytes.
* Decoding stops at the first character from @src that decodes to null.
* Path normalization will remove redundant slashes and slash+dot sequences,
* as well as removing path components when slash+dot+dot is found. It will
* keep the root slash (if one was present) and will stop normalization
* at the first questionmark found (so query parameters won't be normalized).
*
* @param dst destination buffer
* @param src source buffer
* @param normalize perform path normalization if nonzero
* @return number of valid characters in @dst
*/
int urlUnescape(char* dst, const char* src, int normalize)
{
// CURL* curl;
// int resultSize;
// if (curl_global_init(CURL_GLOBAL_ALL) != 0)
// {
// tell(0, "Error, something went wrong with curl_global_init()");
// return fail;
// }
// curl = curl_easy_init();
// if (!curl)
// {
// tell(0, "Error, unable to get handle from curl_easy_init()");
// return fail;
// }
// dst = curl_easy_unescape(curl, src, strlen(src), &resultSize);
// tell(0, " [%.40s]", src);
// tell(0, "res size %d [%.40s]", resultSize, dst);
// return resultSize;
char* org_dst = dst;
int slash_dot_dot = 0;
char ch, a, b;
a = 0;
do {
ch = *src++;
if (ch == '%' && isxdigit(a = src[0]) && isxdigit(b = src[1]))
{
if (a < 'A')
a -= '0';
else if
(a < 'a') a -= 'A' - 10;
else
a -= 'a' - 10;
if (b < 'A')
b -= '0';
else if (b < 'a')
b -= 'A' - 10;
else
b -= 'a' - 10;
ch = 16 * a + b;
src += 2;
}
if (normalize)
{
switch (ch)
{
case '/': // compress consecutive slashes and remove slash-dot
if (slash_dot_dot < 3)
{
dst -= slash_dot_dot;
slash_dot_dot = 1;
break;
}
// fall-through
case '?': // at start of query, stop normalizing
if (ch == '?')
normalize = 0;
// fall-through
case '\0': // remove trailing slash-dot-(dot)
if (slash_dot_dot > 1)
{
dst -= slash_dot_dot;
// remove parent directory if it was two dots
if (slash_dot_dot == 3)
while (dst > org_dst && *--dst != '/')
; // empty body
slash_dot_dot = (ch == '/') ? 1 : 0;
// keep the root slash if any
if (!slash_dot_dot && dst == org_dst && *dst == '/')
++dst;
}
break;
case '.':
if (slash_dot_dot == 1 || slash_dot_dot == 2)
{
++slash_dot_dot;
break;
}
// fall-through
default:
slash_dot_dot = 0;
}
}
*dst++ = ch;
} while(ch);
return (dst - org_dst) - 1;
}

View File

@ -10,6 +10,7 @@
#include <stdint.h> // uint_64_t #include <stdint.h> // uint_64_t
#include <stdlib.h> #include <stdlib.h>
#include <string.h>
#include <string> #include <string>
#include <openssl/md5.h> // MD5_* #include <openssl/md5.h> // MD5_*
@ -25,7 +26,7 @@
#endif #endif
//*************************************************************************** //***************************************************************************
// // Misc
//*************************************************************************** //***************************************************************************
#ifndef VDR_PLUGIN #ifndef VDR_PLUGIN
@ -56,21 +57,67 @@ enum Misc
tmeSecondsPerDay = 24 * tmeSecondsPerHour tmeSecondsPerDay = 24 * tmeSecondsPerHour
}; };
enum Case
{
cUpper,
cLower
};
const char* toCase(Case cs, char* str);
//*************************************************************************** //***************************************************************************
// Tell // Tell
//*************************************************************************** //***************************************************************************
void tell(int eloquence, const char* format, ...); void __attribute__ ((format(printf, 2, 3))) tell(int eloquence, const char* format, ...);
//*************************************************************************** //***************************************************************************
// MemoryStruct for curl callbacks //
//***************************************************************************
char* srealloc(void* ptr, size_t size);
//***************************************************************************
// MemoryStruct
//*************************************************************************** //***************************************************************************
struct MemoryStruct struct MemoryStruct
{ {
MemoryStruct() { memory = 0; clear(); } public:
MemoryStruct() { expireAt = time(0); memory = 0; clear(); }
MemoryStruct(const MemoryStruct* o)
{
size = o->size;
memory = (char*)malloc(size);
memcpy(memory, o->memory, size);
copyAttributes(o);
}
void copyAttributes(const MemoryStruct* o)
{
strcpy(tag, o->tag);
strcpy(name, o->name);
strcpy(contentType, o->contentType);
strcpy(contentEncoding, o->contentEncoding);
strcpy(mimeType, o->mimeType);
headerOnly = o->headerOnly;
modTime = o->modTime;
expireAt = o->expireAt;
}
~MemoryStruct() { clear(); } ~MemoryStruct() { clear(); }
int append(const char* buf, int len)
{
memory = srealloc(memory, size+len);
memcpy(memory+size, buf, len);
size += len;
return success;
}
// data // data
char* memory; char* memory;
@ -80,7 +127,12 @@ struct MemoryStruct
char tag[100]; // the tag to be compared char tag[100]; // the tag to be compared
char name[100]; // content name (filename) char name[100]; // content name (filename)
char contentType[100]; // e.g. text/html
char mimeType[100]; //
char contentEncoding[100]; //
int headerOnly; int headerOnly;
time_t modTime;
time_t expireAt;
int isEmpty() { return memory == 0; } int isEmpty() { return memory == 0; }
@ -91,6 +143,11 @@ struct MemoryStruct
size = 0; size = 0;
*tag = 0; *tag = 0;
*name = 0; *name = 0;
*contentType = 0;
*contentEncoding = 0;
*mimeType = 0;
modTime = time(0);
// !!!! expireAt = time(0);
headerOnly = no; headerOnly = no;
} }
}; };
@ -99,6 +156,7 @@ struct MemoryStruct
// Tools // Tools
//*************************************************************************** //***************************************************************************
double usNow();
unsigned int getHostId(); unsigned int getHostId();
const char* getHostName(); const char* getHostName();
const char* getFirstIp(); const char* getFirstIp();
@ -120,14 +178,23 @@ std::string num2Str(int num);
std::string l2pTime(time_t t); std::string l2pTime(time_t t);
std::string ms2Dur(uint64_t t); std::string ms2Dur(uint64_t t);
const char* c2s(char c, char* buf); const char* c2s(char c, char* buf);
int urlUnescape(char* dst, const char* src, int normalize = yes);
int storeToFile(const char* filename, const char* data, int size);
int loadFromFile(const char* infile, MemoryStruct* data);
int fileExists(const char* path); int fileExists(const char* path);
int fileSize(const char* path); int fileSize(const char* path);
time_t fileModTime(const char* path);
int createLink(const char* link, const char* dest, int force); int createLink(const char* link, const char* dest, int force);
int isLink(const char* path); int isLink(const char* path);
const char* suffixOf(const char* path);
int isEmpty(const char* str); int isEmpty(const char* str);
const char* notNull(const char* str);
int isZero(const char* str);
int removeFile(const char* filename); int removeFile(const char* filename);
int chkDir(const char* path); int chkDir(const char* path);
int downloadFile(const char* url, int& size, MemoryStruct* data, int timeout = 30, const char* userAgent = "libcurl-agent/1.0");
#ifdef USELIBXML #ifdef USELIBXML
xsltStylesheetPtr loadXSLT(const char* name, const char* path, int utf8); xsltStylesheetPtr loadXSLT(const char* name, const char* path, int utf8);
@ -145,6 +212,7 @@ int chkDir(const char* path);
//*************************************************************************** //***************************************************************************
int gunzip(MemoryStruct* zippedData, MemoryStruct* unzippedData); int gunzip(MemoryStruct* zippedData, MemoryStruct* unzippedData);
int gzip(MemoryStruct* data, MemoryStruct* zippedData);
void tellZipError(int errorCode, const char* op, const char* msg); void tellZipError(int errorCode, const char* op, const char* msg);
#ifdef USELIBARCHIVE #ifdef USELIBARCHIVE

View File

@ -44,10 +44,13 @@ cEPG2VDRConfig::cEPG2VDRConfig(void)
scheduleBoot = no; scheduleBoot = no;
#else #else
sstrcpy(cachePath, "/var/cache/epgd", sizeof(cachePath)); sstrcpy(cachePath, "/var/cache/epgd", sizeof(cachePath));
sstrcpy(httpPath, "/var/epgd/www", sizeof(httpPath));
sstrcpy(pluginPath, PLGDIR, sizeof(pluginPath)); sstrcpy(pluginPath, PLGDIR, sizeof(pluginPath));
sstrcpy(epgView, "eventsview.sql", sizeof(epgView)); sstrcpy(epgView, "eventsview.sql", sizeof(epgView));
sstrcpy(theTvDBView, "thetvdbview.sql", sizeof(epgView));
updateThreshold = 200; updateThreshold = 200;
maintanance = no; maintanance = no;
httpPort = 9999;
#endif #endif
sstrcpy(dbHost, "localhost", sizeof(dbHost)); sstrcpy(dbHost, "localhost", sizeof(dbHost));
@ -60,4 +63,7 @@ cEPG2VDRConfig::cEPG2VDRConfig(void)
loglevel = 1; loglevel = 1;
uuid[0] = 0; uuid[0] = 0;
scrapEpg = yes;
scrapRecordings = yes;
} }

View File

@ -47,10 +47,13 @@ struct cEPG2VDRConfig
int scheduleBoot; int scheduleBoot;
#else #else
char cachePath[256+TB]; char cachePath[256+TB];
char httpPath[256+TB];
char pluginPath[256+TB]; char pluginPath[256+TB];
char epgView[100+TB]; char epgView[100+TB];
char theTvDBView[100+TB];
int updateThreshold; int updateThreshold;
int maintanance; int maintanance;
int httpPort;
#endif #endif
char dbHost[100+TB]; char dbHost[100+TB];
@ -66,6 +69,10 @@ struct cEPG2VDRConfig
int mainmenuFullupdate; int mainmenuFullupdate;
int masterMode; int masterMode;
char uuid[sizeUuid+TB]; char uuid[sizeUuid+TB];
int scrapEpg;
int scrapRecordings;
}; };
extern cEPG2VDRConfig EPG2VDRConfig; extern cEPG2VDRConfig EPG2VDRConfig;

122
lib/db.c
View File

@ -7,7 +7,7 @@
#include <stdio.h> #include <stdio.h>
#include <mysql/errmsg.h> #include <errmsg.h>
#include "db.h" #include "db.h"
@ -17,6 +17,8 @@
// DB Statement // DB Statement
//*************************************************************************** //***************************************************************************
int cDbStatement::explain = no;
cDbStatement::cDbStatement(cDbTable* aTable) cDbStatement::cDbStatement(cDbTable* aTable)
{ {
table = aTable; table = aTable;
@ -30,6 +32,14 @@ cDbStatement::cDbStatement(cDbTable* aTable)
affected = 0; affected = 0;
metaResult = 0; metaResult = 0;
bindPrefix = 0; bindPrefix = 0;
firstExec = yes;
callsPeriod = 0;
callsTotal = 0;
duration = 0;
if (connection)
connection->statements.append(this);
} }
cDbStatement::cDbStatement(cDbConnection* aConnection, const char* stmt) cDbStatement::cDbStatement(cDbConnection* aConnection, const char* stmt)
@ -45,6 +55,22 @@ cDbStatement::cDbStatement(cDbConnection* aConnection, const char* stmt)
affected = 0; affected = 0;
metaResult = 0; metaResult = 0;
bindPrefix = 0; bindPrefix = 0;
firstExec = yes;
callsPeriod = 0;
callsTotal = 0;
duration = 0;
if (connection)
connection->statements.append(this);
}
cDbStatement::~cDbStatement()
{
if (connection)
connection->statements.remove(this);
clear();
} }
//*************************************************************************** //***************************************************************************
@ -61,11 +87,43 @@ int cDbStatement::execute(int noResult)
if (!stmt) if (!stmt)
return connection->errorSql(connection, "execute(missing statement)"); return connection->errorSql(connection, "execute(missing statement)");
// if (explain && firstExec)
// {
// firstExec = no;
// if (strstr(stmtTxt.c_str(), "select "))
// {
// MYSQL_RES* result;
// MYSQL_ROW row;
// string q = "explain " + stmtTxt;
// if (connection->query(q.c_str()) != success)
// connection->errorSql(connection, "explain ", 0);
// else if ((result = mysql_store_result(connection->getMySql())))
// {
// while ((row = mysql_fetch_row(result)))
// {
// tell(0, "EXPLAIN: %s) %s %s %s %s %s %s %s %s %s",
// row[0], row[1], row[2], row[3],
// row[4], row[5], row[6], row[7], row[8], row[9]);
// }
// mysql_free_result(result);
// }
// }
// }
// tell(0, "execute %d [%s]", stmt, stmtTxt.c_str()); // tell(0, "execute %d [%s]", stmt, stmtTxt.c_str());
long start = usNow();
if (mysql_stmt_execute(stmt)) if (mysql_stmt_execute(stmt))
return connection->errorSql(connection, "execute(stmt_execute)", stmt, stmtTxt.c_str()); return connection->errorSql(connection, "execute(stmt_execute)", stmt, stmtTxt.c_str());
duration += usNow() - start;
callsPeriod++;
callsTotal++;
// out binding - if needed // out binding - if needed
if (outCount && !noResult) if (outCount && !noResult)
@ -270,7 +328,7 @@ int cDbStatement::appendBinding(cDbValue* value, BindType bt)
if (!bindings) if (!bindings)
*bindings = (MYSQL_BIND*)malloc(count * sizeof(MYSQL_BIND)); *bindings = (MYSQL_BIND*)malloc(count * sizeof(MYSQL_BIND));
else else
*bindings = (MYSQL_BIND*)realloc(*bindings, count * sizeof(MYSQL_BIND)); *bindings = (MYSQL_BIND*)srealloc(*bindings, count * sizeof(MYSQL_BIND));
newBinding = &((*bindings)[count-1]); newBinding = &((*bindings)[count-1]);
@ -360,6 +418,22 @@ int cDbStatement::prepare()
return success; return success;
} }
//***************************************************************************
// Show Statistic
//***************************************************************************
void cDbStatement::showStat()
{
if (callsPeriod)
{
tell(0, "calls %4ld in %6.2fms; total %4ld [%s]",
callsPeriod, duration/1000, callsTotal, stmtTxt.c_str());
callsPeriod = 0;
duration = 0;
}
}
//*************************************************************************** //***************************************************************************
// cDbService // cDbService
//*************************************************************************** //***************************************************************************
@ -382,6 +456,50 @@ const char* cDbService::toString(FieldFormat t)
return formats[t]; return formats[t];
} }
const char* cDbService::dictFormats[] =
{
"int",
"uint",
"ascii",
"text",
"mlob",
"float",
"datetime",
0
};
cDbService::FieldFormat cDbService::toDictFormat(const char* format)
{
for (int i = 0; i < ffCount; i++)
if (strcasecmp(dictFormats[i], format) == 0)
return (FieldFormat)i;
return ffUnknown;
}
const char* cDbService::types[] =
{
"data",
"primary",
"meta",
"calc",
"autoinc",
0
};
cDbService::FieldType cDbService::toType(const char* type)
{
// #TODO !!!
for (int i = 0; i < 3; i++)
if (strcasecmp(types[i], type) == 0)
return (FieldType)i;
return ftUnknown;
}
//*************************************************************************** //***************************************************************************
// Class cDbTable // Class cDbTable
//*************************************************************************** //***************************************************************************

View File

@ -13,11 +13,11 @@
#include <stdio.h> #include <stdio.h>
#include <stdarg.h> #include <stdarg.h>
#include <errno.h> #include <errno.h>
#include <sstream>
#include <mysql/mysql.h> #include <mysql/mysql.h>
#include <list>
#include <sstream>
#include "common.h" #include "common.h"
class cDbTable; class cDbTable;
@ -40,6 +40,8 @@ class cDbService
enum FieldFormat enum FieldFormat
{ {
ffUnknown = na,
ffInt, ffInt,
ffUInt, ffUInt,
ffAscii, // -> VARCHAR ffAscii, // -> VARCHAR
@ -52,6 +54,8 @@ class cDbService
enum FieldType enum FieldType
{ {
ftUnknown = na,
ftData = 1, ftData = 1,
ftPrimary = 2, ftPrimary = 2,
ftMeta = 4, ftMeta = 4,
@ -90,7 +94,12 @@ class cDbService
}; };
static const char* toString(FieldFormat t); static const char* toString(FieldFormat t);
static FieldFormat toDictFormat(const char* format);
static const char* formats[]; static const char* formats[];
static const char* dictFormats[];
static FieldType toType(const char* type);
static const char* types[];
}; };
typedef cDbService cDBS; typedef cDbService cDBS;
@ -374,8 +383,7 @@ class cDbStatement : public cDbService
cDbStatement(cDbTable* aTable); cDbStatement(cDbTable* aTable);
cDbStatement(cDbConnection* aConnection, const char* stmt = ""); cDbStatement(cDbConnection* aConnection, const char* stmt = "");
virtual ~cDbStatement();
virtual ~cDbStatement() { clear(); }
int execute(int noResult = no); int execute(int noResult = no);
int find(); int find();
@ -404,6 +412,12 @@ class cDbStatement : public cDbService
int getResultCount(); int getResultCount();
const char* asText() { return stmtTxt.c_str(); } const char* asText() { return stmtTxt.c_str(); }
void showStat();
// data
static int explain; // debug explain
private: private:
void clear(); void clear();
@ -420,6 +434,44 @@ class cDbStatement : public cDbService
MYSQL_BIND* outBind; // from db (result) MYSQL_BIND* outBind; // from db (result)
MYSQL_RES* metaResult; MYSQL_RES* metaResult;
const char* bindPrefix; const char* bindPrefix;
int firstExec; // debug explain
unsigned long callsPeriod;
unsigned long callsTotal;
double duration;
};
//***************************************************************************
// cDbStatements
//***************************************************************************
class cDbStatements
{
public:
cDbStatements() { statisticPeriod = time(0); }
~cDbStatements() {};
void append(cDbStatement* s) { statements.push_back(s); }
void remove(cDbStatement* s) { statements.remove(s); }
void showStat()
{
tell(0, "Statement statistic of last %ld seconds:", statisticPeriod);
for (std::list<cDbStatement*>::iterator it = statements.begin() ; it != statements.end(); ++it)
{
if (*it)
(*it)->showStat();
}
statisticPeriod = time(0);
}
private:
time_t statisticPeriod;
std::list<cDbStatement*> statements;
}; };
//*************************************************************************** //***************************************************************************
@ -628,7 +680,7 @@ class cDbConnection
{ {
nread += res; nread += res;
size += 1000; size += 1000;
buffer = (char*)realloc(buffer, size+1); buffer = srealloc(buffer, size+1);
} }
fclose(f); fclose(f);
@ -704,6 +756,8 @@ class cDbConnection
int errorSql(cDbConnection* mysql, const char* prefix, MYSQL_STMT* stmt = 0, const char* stmtTxt = 0); int errorSql(cDbConnection* mysql, const char* prefix, MYSQL_STMT* stmt = 0, const char* stmtTxt = 0);
void showStat() { statements.showStat(); }
static int init() static int init()
{ {
if (mysql_library_init(0, 0, 0)) if (mysql_library_init(0, 0, 0))
@ -729,6 +783,8 @@ class cDbConnection
MYSQL* mysql; MYSQL* mysql;
cDbStatements statements; // all statements of this connection
private: private:
int initialized; int initialized;

View File

@ -19,8 +19,8 @@ const char* cEpgdState::states[] =
"busy (events)", "busy (events)",
"busy (match)", "busy (match)",
"busy (images)",
"busy (scraping)", "busy (scraping)",
"busy (images)",
0 0
}; };

View File

@ -26,14 +26,17 @@ class cEpgdState
esStandby, esStandby,
esStopped, esStopped,
// handler pause on this states!
esBusy, esBusy,
esBusyEvents = esBusy, esBusyEvents = esBusy,
esBusyMatch, esBusyMatch,
esBusyScraping,
// handler don't pause on this states!
esBusyImages, esBusyImages,
esBusyScraping,
esCount esCount
}; };
@ -78,6 +81,7 @@ class cUpdateState
static const char* getDeletable() { return "'A','L','P','R','I'"; } static const char* getDeletable() { return "'A','L','P','R','I'"; }
static const char* getNeeded() { return "'A','L','P','C','D','R'"; } static const char* getNeeded() { return "'A','L','P','C','D','R'"; }
static const char* getVisible() { return "'A','L','P'"; }
// checks fpr c++ code // checks fpr c++ code

View File

@ -13,7 +13,7 @@
//*************************************************************************** //***************************************************************************
// Constants // Constants
//*************************************************************************** //***************************************************************************
static const char *VERSION = "0.1.1"; static const char *VERSION = "0.1.2";
static const char *DESCRIPTION = "'scraper2vdr' plugin"; static const char *DESCRIPTION = "'scraper2vdr' plugin";
static const char *MAINMENUENTRY = "Scraper2Vdr"; static const char *MAINMENUENTRY = "Scraper2Vdr";

View File

@ -354,7 +354,7 @@ void cScrapManager::DumpMovies(void) {
} }
void cScrapManager::DumpRecordings(void) { void cScrapManager::DumpRecordings(void) {
tell(0, "%d recordings in memory:", recordings.size()); tell(0, "%ld recordings in memory:", recordings.size());
for (map<sRecordingsKey, sEventsValue>::iterator it = recordings.begin(); it != recordings.end(); it++) { for (map<sRecordingsKey, sEventsValue>::iterator it = recordings.begin(); it != recordings.end(); it++) {
sRecordingsKey key = it->first; sRecordingsKey key = it->first;
sEventsValue val = it->second; sEventsValue val = it->second;
@ -373,6 +373,7 @@ bool cScrapManager::GetEventType(ScraperGetEventType *call) {
tell(0, "scraper2vdr plugin service call GetEventType called"); tell(0, "scraper2vdr plugin service call GetEventType called");
} }
sEventsValue v; sEventsValue v;
memset(&v, 0, sizeof(sEventsValue));
if (call->event) { if (call->event) {
sEventsKey k; sEventsKey k;
k.eventId = call->event->EventID(); k.eventId = call->event->EventID();
@ -543,6 +544,7 @@ bool cScrapManager::GetPoster(ScraperGetPoster *call) {
return m->GetMedia(mmPoster, &call->poster); return m->GetMedia(mmPoster, &call->poster);
} }
return true;
} }
bool cScrapManager::GetPosterThumb(ScraperGetPosterThumb *call) { bool cScrapManager::GetPosterThumb(ScraperGetPosterThumb *call) {

View File

@ -1,3 +1,6 @@
#include "lib/config.h"
#include "setup.h" #include "setup.h"
extern cScraper2VdrConfig config; extern cScraper2VdrConfig config;
@ -39,7 +42,7 @@ void cScraper2VdrSetup::Setup(void) {
} }
eOSState cScraper2VdrSetup::ProcessKey(eKeys Key) { eOSState cScraper2VdrSetup::ProcessKey(eKeys Key) {
bool hadSubMenu = HasSubMenu(); // bool hadSubMenu = HasSubMenu();
eOSState state = cMenuSetupPage::ProcessKey(Key); eOSState state = cMenuSetupPage::ProcessKey(Key);
if (Key == kOk) { if (Key == kOk) {
tmpConfig.mysqlHost = host; tmpConfig.mysqlHost = host;
@ -77,4 +80,6 @@ void cScraper2VdrSetup::Store(void) {
SetupStore("mysqlDBUser", tmpConfig.mysqlDBUser.c_str()); SetupStore("mysqlDBUser", tmpConfig.mysqlDBUser.c_str());
SetupStore("mysqlDBPass", tmpConfig.mysqlDBPass.c_str()); SetupStore("mysqlDBPass", tmpConfig.mysqlDBPass.c_str());
SetupStore("debug", tmpConfig.debug); SetupStore("debug", tmpConfig.debug);
EPG2VDRConfig.loglevel = tmpConfig.debug ? 2 : 1;
} }

View File

@ -144,7 +144,7 @@ vector<string>& splitstring::split(char delim, int rep) {
if (!flds.empty()) flds.clear(); // empty vector if necessary if (!flds.empty()) flds.clear(); // empty vector if necessary
string work = data(); string work = data();
string buf = ""; string buf = "";
int i = 0; unsigned int i = 0;
while (i < work.length()) { while (i < work.length()) {
if (work[i] != delim) if (work[i] != delim)
buf += work[i]; buf += work[i];

827
update.c

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,6 @@
#ifndef __UPDATE_H #ifndef __UPDATE_H
#define __UPDATE_H #define __UPDATE_H
#include <mysql/mysql.h>
#include <map> #include <map>
#include <vdr/thread.h> #include <vdr/thread.h>
@ -71,6 +70,33 @@ class cUpdate : public cThread {
int CleanupSeries(void); int CleanupSeries(void);
int CleanupMovies(void); int CleanupMovies(void);
int CleanupRecordings(void); int CleanupRecordings(void);
// statements
cDbStatement* selectReadScrapedEventsInit;
cDbStatement* selectReadScrapedEvents;
cDbStatement* selectImg;
cDbStatement* selectSeasonPoster;
cDbStatement* selectActors;
cDbStatement* selectActorThumbs;
cDbStatement* selectSeriesMedia;
cDbStatement* selectMovieActors;
cDbStatement* selectMovieActorThumbs;
cDbStatement* selectMovieMedia;
cDbStatement* selectMediaMovie;
cDbStatement* selectRecordings;
cDbStatement* selectCleanupRecordings;
cDbValue imageSize;
cDbValue posterSize;
cDbValue series_id;
cDbValue actorImageSize;
cDbValue actorRole;
cDbValue actorMovie;
cDbValue thbWidth;
cDbValue thbHeight;
public: public:
cUpdate(cScrapManager *manager); cUpdate(cScrapManager *manager);
virtual ~cUpdate(void); virtual ~cUpdate(void);