Implemented EPG bugfixing

HISTORY

@@ -650,8 +650,13 @@ Video Disk Recorder Revision History
   only once.
 - Made I/O more robust by handling EINTR (thanks to Werner Fink).
 
-2001-08-15: Version 0.92
+2001-08-17: Version 0.92
 
 - The "channel not sync'ed" log message now also lists the card number.
 - Now using the EIT services from 'libdtv' (thanks to Rolf Hakenes), which
   provides EPG information for NVOD ("Near Video On Demand") channels.
+- Doing some bug fixing on the EPG data (some tv stations apparently have
+  their own idea on how to fill in the data...). The level up to which EPG
+  bugs are fixed can be controlled with the EPGBugfixLevel parameter in the
+  "Setup" menu (see MANUAL for details, and cEventInfo::FixEpgBugs() in eit.c
+  for the actual implementation).

MANUAL

@@ -351,6 +351,24 @@ Video Disk Recorder User's Manual
                          A value of '0' completely turns off scanning on both single
                          and multiple card systems.
 
+  EPGBugfixLevel = 2     Some tv stations transmit weirdly formatted EPG data.
+                         VDR attempts to fix these bugs up to the given level:
+                         0 = no EPG fixing
+                         1 = basic fixing of text location (Title, Subtitle and
+                             Extended Description)
+                         2 = removal of excess whitespace and hyphens
+                         3 = fixing the date in timestamps between 00:00 and 06:00
+                             (use with care - hopefully one day Pro7 and Kabel1
+                             will learn how to read the clock/calender)
+                         Default is '2', which will do all textual fixes, but
+                         leaves out the timestamp fixes, since these might cause
+                         recordings to fail. Use '3' at your own risk.
+                         Note that after changing the setting of this parameter
+                         any EPG data that has already been received will remain
+                         in its existing format - only newly received data will
+                         be fixed accordingly. Restart VDR if you want to make sure
+                         all data is fixed.
+
   SVDRPTimeout = 300     The time (in seconds) of inactivity on an open SVDRP
                          connection after which the connection is automatically
                          closed. Default is 300, a value of 0 means no timeout.

config.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: config.c 1.54 2001/08/11 15:34:42 kls Exp $
+ * $Id: config.c 1.55 2001/08/17 13:02:01 kls Exp $
  */
 
 #include "config.h"
@@ -774,6 +774,7 @@ cSetup::cSetup(void)
   MarginStart = 2;
   MarginStop = 10;
   EPGScanTimeout = 5;
+  EPGBugfixLevel = 2;
   SVDRPTimeout = 300;
   PrimaryLimit = 0;
   DefaultPriority = 50;
@@ -804,6 +805,7 @@ bool cSetup::Parse(char *s)
      else if (!strcasecmp(Name, "MarginStart"))         MarginStart        = atoi(Value);
      else if (!strcasecmp(Name, "MarginStop"))          MarginStop         = atoi(Value);
      else if (!strcasecmp(Name, "EPGScanTimeout"))      EPGScanTimeout     = atoi(Value);
+     else if (!strcasecmp(Name, "EPGBugfixLevel"))      EPGBugfixLevel     = atoi(Value);
      else if (!strcasecmp(Name, "SVDRPTimeout"))        SVDRPTimeout       = atoi(Value);
      else if (!strcasecmp(Name, "PrimaryLimit"))        PrimaryLimit       = atoi(Value);
      else if (!strcasecmp(Name, "DefaultPriority"))     DefaultPriority    = atoi(Value);
@@ -869,6 +871,7 @@ bool cSetup::Save(const char *FileName)
         fprintf(f, "MarginStart        = %d\n", MarginStart);
         fprintf(f, "MarginStop         = %d\n", MarginStop);
         fprintf(f, "EPGScanTimeout     = %d\n", EPGScanTimeout);
+        fprintf(f, "EPGBugfixLevel     = %d\n", EPGBugfixLevel);
         fprintf(f, "SVDRPTimeout       = %d\n", SVDRPTimeout);
         fprintf(f, "PrimaryLimit       = %d\n", PrimaryLimit);
         fprintf(f, "DefaultPriority    = %d\n", DefaultPriority);

config.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: config.h 1.60 2001/08/15 09:24:10 kls Exp $
+ * $Id: config.h 1.61 2001/08/17 13:00:48 kls Exp $
  */
 
 #ifndef __CONFIG_H
@@ -280,6 +280,7 @@ public:
   int SetSystemTime;
   int MarginStart, MarginStop;
   int EPGScanTimeout;
+  int EPGBugfixLevel;
   int SVDRPTimeout;
   int PrimaryLimit;
   int DefaultPriority, DefaultLifetime;

eit.c

@@ -16,7 +16,7 @@
  *   the Free Software Foundation; either version 2 of the License, or     *
  *   (at your option) any later version.                                   *
  *                                                                         *
- * $Id: eit.c 1.20 2001/08/15 13:23:21 kls Exp $
+ * $Id: eit.c 1.21 2001/08/17 13:19:10 kls Exp $
  ***************************************************************************/
 
 #include "eit.h"
@@ -190,7 +190,6 @@ cEventInfo::cEventInfo(unsigned short serviceid, unsigned short eventid)
    tTime = 0;
    uEventID = eventid;
    uServiceID = serviceid;
-   cExtendedDescriptorNumber = 0;
    nChannelNumber = 0;
 }
 
@@ -354,16 +353,6 @@ void cEventInfo::SetServiceID(unsigned short servid)
 {
    uServiceID = servid;
 }
-/**  */
-u_char cEventInfo::GetExtendedDescriptorNumber() const
-{
-   return cExtendedDescriptorNumber;
-}
-/**  */
-void cEventInfo::IncreaseExtendedDescriptorNumber()
-{
-   cExtendedDescriptorNumber++;
-}
 
 /**  */
 unsigned short cEventInfo::GetServiceID() const
@@ -386,6 +375,116 @@ void cEventInfo::Dump(FILE *f, const char *Prefix) const
       }
 }
 
+void cEventInfo::FixEpgBugs(void)
+{
+  if (Setup.EPGBugfixLevel == 0)
+     return;
+
+  // Some TV stations apparently have their own idea about how to fill in the
+  // EPG data. Let's fix their bugs as good as we can:
+  if (pTitle) {
+
+     // Pro7 preceeds the Subtitle with the Title:
+     //
+     // Title
+     // Title / Subtitle
+     //
+     if (pSubtitle && strstr(pSubtitle, pTitle) == pSubtitle) {
+        char *p = pSubtitle + strlen(pTitle);
+        const char *delim = " / ";
+        if (strstr(p, delim) == p) {
+           p += strlen(delim);
+           memmove(pSubtitle, p, strlen(p) + 1);
+           }
+        }
+
+     // VOX and VIVA put the Subtitle in quotes and use either the Subtitle
+     // or the Extended Description field, depending on how long the string is:
+     //
+     // Title
+     // "Subtitle". Extended Description
+     //
+     if ((pSubtitle == NULL) != (pExtendedDescription == NULL)) {
+        char *p = pSubtitle ? pSubtitle : pExtendedDescription;
+        if (*p == '"') {
+           const char *delim = "\".";
+           char *e = strstr(p + 1, delim);
+           if (e) {
+              *e = 0;
+              char *s = strdup(p + 1);
+              char *d = strdup(e + strlen(delim));
+              delete pSubtitle;
+              delete pExtendedDescription;
+              pSubtitle = s;
+              pExtendedDescription = d;
+              }
+           }
+        }
+
+     // VOX and VIVA put the Extended Description into the Subtitle (preceeded
+     // by a blank) if there is no actual Subtitle and the Extended Description
+     // is short enough:
+     //
+     // Title
+     //  Extended Description
+     //
+     if (pSubtitle && !pExtendedDescription) {
+        if (*pSubtitle == ' ') {
+           memmove(pSubtitle, pSubtitle + 1, strlen(pSubtitle));
+           pExtendedDescription = pSubtitle;
+           pSubtitle = NULL;
+           }
+        }
+     }
+
+     // Pro7 sometimes repeats the Title in the Subtitle:
+     //
+     // Title
+     // Title
+     //
+     if (pSubtitle && strcmp(pTitle, pSubtitle) == 0) {
+        delete pSubtitle;
+        pSubtitle = NULL;
+        }
+
+     if (Setup.EPGBugfixLevel <= 1)
+        return;
+
+     // Some channels apparently try to do some formatting in the texts,
+     // which is a bad idea because they have no way of knowing the width
+     // of the window that will actually display the text.
+     // Remove excess whitespace:
+     pTitle = compactspace(pTitle);
+     pSubtitle = compactspace(pSubtitle);
+     pExtendedDescription = compactspace(pExtendedDescription);
+     // Remove superfluous hyphens:
+     if (pExtendedDescription) {
+        char *p = pExtendedDescription + 1;
+        while (*p) {
+              if (*p == '-' && *(p + 1) == ' ' && *(p + 2) && islower(*(p - 1)) && islower(*(p + 2))) {
+                 if (!startswith(p + 2, "und ")) // special case in German, as in "Lach- und Sachgeschichten"
+                    memmove(p, p + 2, strlen(p + 2) + 1);
+                 }
+              p++;
+              }
+        }
+
+     if (Setup.EPGBugfixLevel <= 2)
+        return;
+
+     // Pro7 and Kabel1 apparently are unable to use a calendar/clock,
+     // because all events between 00:00 and 06:00 have the date of the
+     // day before (sometimes even this correction doesn't help).
+     // Channels are recognized by their ServiceID, which may only work
+     // correctly on the ASTRA satellite system.
+     if (uServiceID == 898    // Pro-7
+      || uServiceID == 899) { // Kabel 1
+        tm *t = localtime(&tTime);
+        if (t->tm_hour * 3600 + t->tm_min * 60 + t->tm_sec <= 6 * 3600)
+           tTime += 24 * 3600;
+        }
+}
+
 // --- cSchedule -------------------------------------------------------------
 
 cSchedule::cSchedule(unsigned short servid)
@@ -677,18 +776,17 @@ int cEIT::ProcessEIT(unsigned char *buffer)
                 pEvent->SetSubtitle(rEvent->GetSubtitle());
                 pEvent->SetTime(VdrProgramInfo->StartTime);
                 pEvent->SetDuration(VdrProgramInfo->Duration);
-                if (pEvent->AddExtendedDescription(rEvent->GetExtendedDescription()))
+                pEvent->AddExtendedDescription(rEvent->GetExtendedDescription());
-                   pEvent->IncreaseExtendedDescriptorNumber();
+                pEvent->FixEpgBugs();
                 }
              else {
                 pEvent->SetTitle(VdrProgramInfo->ShortName);
                 pEvent->SetSubtitle(VdrProgramInfo->ShortText);
                 pEvent->SetTime(VdrProgramInfo->StartTime);
                 pEvent->SetDuration(VdrProgramInfo->Duration);
-                if (pEvent->AddExtendedDescription(VdrProgramInfo->ExtendedName))
+                pEvent->AddExtendedDescription(VdrProgramInfo->ExtendedName);
-                   pEvent->IncreaseExtendedDescriptorNumber();
+                pEvent->AddExtendedDescription(VdrProgramInfo->ExtendedText);
-                if (pEvent->AddExtendedDescription(VdrProgramInfo->ExtendedText))
+                pEvent->FixEpgBugs();
-                   pEvent->IncreaseExtendedDescriptorNumber();
                 }
              }
           if (IsPresentFollowing()) {

eit.h

@@ -16,7 +16,7 @@
  *   the Free Software Foundation; either version 2 of the License, or     *
  *   (at your option) any later version.                                   *
  *                                                                         *
- * $Id: eit.h 1.9 2001/08/15 13:10:28 kls Exp $
+ * $Id: eit.h 1.10 2001/08/15 15:47:31 kls Exp $
  ***************************************************************************/
 
 #ifndef __EIT_H
@@ -38,7 +38,6 @@ private:
   unsigned short uEventID;          // Event ID of this event
   long lDuration;                   // duration of event in seconds
   time_t tTime;                     // Start time
-  u_char cExtendedDescriptorNumber; // current extended descriptor number that has to be inserted
   int nChannelNumber;               // the actual channel number from VDR's channel list (used in cMenuSchedule for sorting by channel number)
 protected:
   void SetFollowing(bool foll);
@@ -50,7 +49,6 @@ protected:
   void SetTime(time_t t);
   bool AddExtendedDescription(const char *string);
   bool SetSubtitle(const char *string);
-  void IncreaseExtendedDescriptorNumber(void);
   cEventInfo(unsigned short serviceid, unsigned short eventid);
 public:
   ~cEventInfo();
@@ -65,11 +63,11 @@ public:
   unsigned short GetEventID(void) const;
   long GetDuration(void) const;
   time_t GetTime(void) const;
-  u_char GetExtendedDescriptorNumber(void) const;
   unsigned short GetServiceID(void) const;
   int GetChannelNumber(void) const { return nChannelNumber; }
   void SetChannelNumber(int ChannelNumber) const { ((cEventInfo *)this)->nChannelNumber = ChannelNumber; } // doesn't modify the EIT data, so it's ok to make it 'const'
   void Dump(FILE *f, const char *Prefix = "") const;
+  void FixEpgBugs(void);
   };
 
 class cSchedule : public cListObject  {

i18n.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: i18n.c 1.31 2001/08/11 13:22:24 kls Exp $
+ * $Id: i18n.c 1.32 2001/08/17 13:03:15 kls Exp $
  *
  * Slovenian translations provided by Miha Setina <mihasetina@softhome.net>
  * Italian   translations provided by Alberto Carraro <bertocar@tin.it>
@@ -776,6 +776,15 @@ const tPhrase Phrases[] = {
     "Temps maxi EPG",
     "Ledig tid før EPG-søk",
   },
+  { "EPGBugfixLevel",
+    "EPG Fehlerbereinigung",
+    "", // TODO
+    "", // TODO
+    "", // TODO
+    "", // TODO
+    "", // TODO
+    "", // TODO
+  },
   { "SVDRPTimeout",
     "SVDRP Timeout",
     "", // TODO

menu.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: menu.c 1.103 2001/08/12 12:42:37 kls Exp $
+ * $Id: menu.c 1.104 2001/08/17 13:02:27 kls Exp $
  */
 
 #include "menu.h"
@@ -1701,6 +1701,7 @@ void cMenuSetup::Set(void)
   Add(new cMenuEditIntItem( tr("MarginStart"),        &data.MarginStart));
   Add(new cMenuEditIntItem( tr("MarginStop"),         &data.MarginStop));
   Add(new cMenuEditIntItem( tr("EPGScanTimeout"),     &data.EPGScanTimeout));
+  Add(new cMenuEditIntItem( tr("EPGBugfixLevel"),     &data.EPGBugfixLevel, 0, 3));
   Add(new cMenuEditIntItem( tr("SVDRPTimeout"),       &data.SVDRPTimeout));
   Add(new cMenuEditIntItem( tr("PrimaryLimit"),       &data.PrimaryLimit, 0, MAXPRIORITY));
   Add(new cMenuEditIntItem( tr("DefaultPriority"),    &data.DefaultPriority, 0, MAXPRIORITY));

tools.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: tools.c 1.39 2001/08/12 15:12:54 kls Exp $
+ * $Id: tools.c 1.40 2001/08/17 12:45:42 kls Exp $
  */
 
 #define _GNU_SOURCE
@@ -103,6 +103,32 @@ char *stripspace(char *s)
   return s;
 }
 
+char *compactspace(char *s)
+{
+  if (s && *s) {
+     char *t = stripspace(skipspace(s));
+     char *p = t;
+     while (p && *p) {
+           char *q = skipspace(p);
+           if (q - p > 1)
+              memmove(p + 1, q, strlen(q) + 1);
+           p++;
+           }
+     if (t != s)
+        memmove(s, t, strlen(t) + 1);
+     }
+  return s;
+}
+
+bool startswith(const char *s, const char *p)
+{
+  while (*p) {
+        if (*p++ != *s++)
+           return false;
+        }
+  return true;
+}
+
 bool isempty(const char *s)
 {
   return !(s && *skipspace(s));

tools.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: tools.h 1.28 2001/08/12 15:13:02 kls Exp $
+ * $Id: tools.h 1.29 2001/08/17 12:44:39 kls Exp $
  */
 
 #ifndef __TOOLS_H
@@ -41,6 +41,8 @@ char *strn0cpy(char *dest, const char *src, size_t n);
 char *strreplace(char *s, char c1, char c2);
 char *skipspace(const char *s);
 char *stripspace(char *s);
+char *compactspace(char *s);
+bool startswith(const char *s, const char *p);
 bool isempty(const char *s);
 int time_ms(void);
 void delay_ms(int ms);