Improved handling EPG data from the EIT tables

HISTORY

@@ -9578,7 +9578,7 @@ Video Disk Recorder Revision History
   given (reported by Manuel Reimer).
 - Fixed handling $(PKG_CONFIG) in newplugin (thanks to Winfried Köhler).
 
-2021-03-17:
+2021-04-04:
 
 - Fixed strreplace() to handle NULL strings (reported by Jürgen Schneider).
 - Somewhere down the road the 'x' bit of Doxyfile.filter got lost, so the
@@ -9617,3 +9617,17 @@ Video Disk Recorder Revision History
 - Decreased the scrambling timeout for CAMs known to decrypt a certain channel, so
   that it won't collide with MAXBROKENTIMEOUT in recorder.c.
 - Fixed scaling subtitles with anti-aliasing (thanks to Peter Bieringer).
+- Improved handling EPG data from the EIT tables:
+  + Table 0x4F is now completely ignored.
+  + Once a schedule has seen events from 0x5X, tables 0x6X are ignored for that
+    schedule.
+  + When looking up an event in its schedule, the start time is used for tables 0x6X, and the
+    event id for tables 0x4E and 0x5X.
+  + When hashing events by event id or start time, existing older entries in the hash
+    tables are now deleted before entering the new ones.
+  + The function cSchedule::GetEvent() is now deprecated and may be removed in a future
+    version. Use GetEventById() and GetEventByTime() instead.
+  + On channels that use proper event ids a change of the start time no longer
+    causes a new event to be created, but rather modifies the existing one. This
+    avoids possible interruptions in VPS recordings in case the event's start time
+    is changed while the recording is already going on.

eit.c

@@ -8,9 +8,26 @@
  * Robert Schneider <Robert.Schneider@web.de> and Rolf Hakenes <hakenes@hippomi.de>.
  * Adapted to 'libsi' for VDR 1.3.0 by Marcel Wiesweg <marcel.wiesweg@gmx.de>.
  *
- * $Id: eit.c 5.1 2021/03/16 15:10:54 kls Exp $
+ * $Id: eit.c 5.2 2021/04/04 11:06:30 kls Exp $
  */
 
+// The various ways in which broadcasters handle (or screw up) their EPG:
+// - Some use the same version for all tables, and use unique event ids, which are the same in
+//   both the 0x5X and the 0x6X tables. And once an event has an id, it keeps it until it is
+//   no longer in the tables. Those are the good guys!
+// - Some use separate versions for each table (0x50, 0x51, ...).
+// - Some broadcast tables 0x5X and 0x6X, but use different event ids for the same event in both
+//   sets of tables, and sometimes even use different titles, short texts or descriptions.
+// - Some broadcast the full EPG only on one transponder (tables 0x6X), and on the actual transponder
+//   they provide only the present/following information.
+// - Some have overlapping events, especially when they mess up daylight saving time.
+// - Some use all new event ids every time they update their tables.
+// So, to bring order to chaos, VDR does as follows:
+// - Completely ignore table 0x4F.
+// - Once a schedule has seen events from 0x5X, tables 0x6X are ignored for that schedule.
+// - When looking up an event in its schedule, the start time is used for tables 0x6X, and the
+//   event id for tables 0x4E and 0x5X.
+
 #include "eit.h"
 #include <sys/time.h>
 #include "epg.h"
@@ -24,6 +41,13 @@
 
 // --- cEitTables ------------------------------------------------------------
 
+cEitTables::cEitTables(void)
+{
+  complete = false;
+  tableStart = 0;
+  tableEnd = 0;
+}
+
 bool cEitTables::Check(uchar TableId, uchar Version, int SectionNumber)
 {
   int ti = Index(TableId);
@@ -32,35 +56,38 @@ bool cEitTables::Check(uchar TableId, uchar Version, int SectionNumber)
 
 bool cEitTables::Processed(uchar TableId, uchar LastTableId, int SectionNumber, int LastSectionNumber, int SegmentLastSectionNumber)
 {
+  bool Result = false;
   int ti = Index(TableId);
   int LastIndex = Index(LastTableId);
+  complete = false;
   if (sectionSyncer[ti].Processed(SectionNumber, LastSectionNumber, SegmentLastSectionNumber)) {
+     Result = true; // the table with TableId is complete
      for (int i = 0; i <= LastIndex; i++) {
          if (!sectionSyncer[i].Complete())
-            return false;
+            return Result;
          }
-     return true; // all tables have been processed
+     complete = true; // all tables have been processed
      }
-  return false;
+  return Result;
 }
 
 // --- cEIT ------------------------------------------------------------------
 
 class cEIT : public SI::EIT {
 public:
-  cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const u_char *Data);
+  cEIT(cEitTablesHash &EitTablesHash, int Source, u_char Tid, const u_char *Data);
   };
 
-cEIT::cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const u_char *Data)
+cEIT::cEIT(cEitTablesHash &EitTablesHash, int Source, u_char Tid, const u_char *Data)
 :SI::EIT(Data, false)
 {
   if (!CheckCRCAndParse())
      return;
   int HashId = getServiceId();
-  cEitTables *EitTables = SectionSyncerHash.Get(HashId);
+  cEitTables *EitTables = EitTablesHash.Get(HashId);
   if (!EitTables) {
      EitTables = new cEitTables;
-     SectionSyncerHash.Add(EitTables, HashId);
+     EitTablesHash.Add(EitTables, HashId);
      }
   bool Process = EitTables->Check(Tid, getVersionNumber(), getSectionNumber());
   if (Tid != 0x4E && !Process) // we need to set the 'seen' tag to watch the running status of the present/following event
@@ -98,10 +125,16 @@ cEIT::cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const
   bool handledExternally = EpgHandlers.HandledExternally(Channel);
   cSchedule *pSchedule = (cSchedule *)Schedules->GetSchedule(Channel, true);
 
+  if (pSchedule->OnActualTp(Tid) && (Tid & 0xF0) == 0x60) {
+     SchedulesStateKey.Remove(false);
+     ChannelsStateKey.Remove(false);
+     return;
+     }
+
   bool Empty = true;
   bool Modified = false;
   time_t LingerLimit = Now - Setup.EPGLinger * 60;
-  time_t SegmentStart = 0;
+  time_t SegmentStart = 0; // these are actually "section" start/end times
   time_t SegmentEnd = 0;
   struct tm t = { 0 };
   localtime_r(&Now, &t); // this initializes the time zone in 't'
@@ -122,9 +155,19 @@ cEIT::cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const
       if (!SegmentStart)
          SegmentStart = StartTime;
       SegmentEnd = StartTime + Duration;
+      if (Tid == 0x4E) {
+         if (getSectionNumber() == 0)
+            EitTables->SetTableStart(SegmentStart);
+         else
+            EitTables->SetTableEnd(SegmentEnd);
+         }
       cEvent *newEvent = NULL;
       cEvent *rEvent = NULL;
-      cEvent *pEvent = (cEvent *)pSchedule->GetEvent(SiEitEvent.getEventId(), StartTime);
+      cEvent *pEvent = NULL;
+      if (Tid == 0x4E || (Tid & 0xF0) == 0x50)
+         pEvent = const_cast<cEvent *>(pSchedule->GetEventById(SiEitEvent.getEventId()));
+      else
+         pEvent = const_cast<cEvent *>(pSchedule->GetEventByTime(StartTime));
       if (!pEvent || handledExternally) {
          if (handledExternally && !EpgHandlers.IsUpdate(SiEitEvent.getEventId(), StartTime, Tid, getVersionNumber()))
             continue;
@@ -140,10 +183,13 @@ cEIT::cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const
          // We have found an existing event, either through its event ID or its start time.
          pEvent->SetSeen();
          uchar TableID = max(pEvent->TableID(), uchar(0x4E)); // for backwards compatibility, table ids less than 0x4E are treated as if they were "present"
-         // If the new event has a higher table ID, let's skip it.
+         // We never overwrite present/following with events from other tables:
-         // The lower the table ID, the more "current" the information.
+         if (TableID == 0x4E && Tid != 0x4E)
-         if (Tid > TableID)
             continue;
+         if (pEvent->HasTimer()) {
+            if (pEvent->StartTime() != StartTime || pEvent->Duration() != Duration)
+               dsyslog("channel %d (%s) event %s times changed to %s-%s", Channel->Number(), Channel->Name(), *pEvent->ToDescr(), *TimeString(StartTime), *TimeString(StartTime + Duration));
+            }
          EpgHandlers.SetEventID(pEvent, SiEitEvent.getEventId()); // unfortunately some stations use different event ids for the same event in different tables :-(
          EpgHandlers.SetStartTime(pEvent, StartTime);
          EpgHandlers.SetDuration(pEvent, Duration);
@@ -275,7 +321,7 @@ cEIT::cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const
                  cSchedule *rSchedule = (cSchedule *)Schedules->GetSchedule(tChannelID(Source, Channel->Nid(), Channel->Tid(), tsed->getReferenceServiceId()));
                  if (!rSchedule)
                     break;
-                 rEvent = (cEvent *)rSchedule->GetEvent(tsed->getReferenceEventId());
+                 rEvent = (cEvent *)rSchedule->GetEventById(tsed->getReferenceEventId());
                  if (!rEvent)
                     break;
                  EpgHandlers.SetTitle(pEvent, rEvent->Title());
@@ -375,13 +421,17 @@ cEIT::cEIT(cSectionSyncerHash &SectionSyncerHash, int Source, u_char Tid, const
         pSchedule->ClrRunningStatus(Channel);
      pSchedule->SetPresentSeen();
      }
-  if (Modified) {
+  if (Process) {
+     bool Complete = EitTables->Processed(Tid, getLastTableId(), getSectionNumber(), getLastSectionNumber(), getSegmentLastSectionNumber());
+     if (Modified && (Tid >= 0x50 || Complete)) { // we process the 0x5X tables segment by segment, but 0x4E only if we have received ALL its segments (0 and 1, i.e. "present" and "following")
+        if (Tid == 0x4E && getLastSectionNumber() == 1) {
+           SegmentStart = EitTables->TableStart();
+           SegmentEnd = EitTables->TableEnd();
+           }
         EpgHandlers.SortSchedule(pSchedule);
         EpgHandlers.DropOutdated(pSchedule, SegmentStart, SegmentEnd, Tid, getVersionNumber());
-     pSchedule->SetModified();
         }
-  if (Process)
+     }
-     EitTables->Processed(Tid, getLastTableId(), getSectionNumber(), getLastSectionNumber(), getSegmentLastSectionNumber());
   SchedulesStateKey.Remove(Modified);
   ChannelsStateKey.Remove(ChannelsModified);
   EpgHandlers.EndSegmentTransfer(Modified);
@@ -443,7 +493,7 @@ time_t cEitFilter::disableUntil = 0;
 
 cEitFilter::cEitFilter(void)
 {
-  Set(0x12, 0x40, 0xC0);  // event info now&next actual/other TS (0x4E/0x4F), future actual/other TS (0x5X/0x6X)
+  Set(0x12, 0x40, 0xC0);  // event info present&following actual/other TS (0x4E/0x4F), future actual/other TS (0x5X/0x6X)
   Set(0x14, 0x70);        // TDT
 }
 
@@ -451,7 +501,7 @@ void cEitFilter::SetStatus(bool On)
 {
   cMutexLock MutexLock(&mutex);
   cFilter::SetStatus(On);
-  sectionSyncerHash.Clear();
+  eitTablesHash.Clear();
 }
 
 void cEitFilter::SetDisableUntil(time_t Time)
@@ -470,8 +520,8 @@ void cEitFilter::Process(u_short Pid, u_char Tid, const u_char *Data, int Length
      }
   switch (Pid) {
     case 0x12: {
-         if (Tid >= 0x4E && Tid <= 0x6F)
+         if (Tid == 0x4E || Tid >= 0x50 && Tid <= 0x6F) // we ignore 0x4F, which only causes trouble
-            cEIT EIT(sectionSyncerHash, Source(), Tid, Data);
+            cEIT EIT(eitTablesHash, Source(), Tid, Data);
          }
          break;
     case 0x14: {

eit.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: eit.h 5.1 2021/03/16 15:10:54 kls Exp $
+ * $Id: eit.h 5.2 2021/04/04 11:06:30 kls Exp $
  */
 
 #ifndef __EIT_H
@@ -17,30 +17,38 @@
 
 // Event information (or EPG) is broadcast in tables 0x4E and 0x4F for "present/following" events on
 // "this transponder" (0x4E) and "other transponders" (0x4F), as well as 0x50-0x5F ("all events on this
-// transponder) and 0x60-0x6F ("all events on other transponders). Since it's either "this" or "other",
+// transponder") and 0x60-0x6F ("all events on other transponders"). Since it's either "this" or "other",
 // we only use one section syncer for 0x4E/0x4F and 16 syncers for either 0x5X or 0x6X.
 
 class cEitTables : public cListObject {
 private:
   cSectionSyncerRandom sectionSyncer[NUM_EIT_TABLES]; // for tables 0x4E/0x4F and 0x50-0x5F/0x60-0x6F
+  time_t tableStart; // only used for table 0x4E
+  time_t tableEnd;
   bool complete;
   int Index(uchar TableId) { return (TableId < 0x50) ? 0 : (TableId & 0x0F) + 1; }
 public:
-  cEitTables(void) { complete = false; }
+  cEitTables(void);
+  void SetTableStart(time_t t) { tableStart = t; }
+  void SetTableEnd(time_t t) { tableEnd = t; }
+  time_t TableStart(void) { return tableStart; }
+  time_t TableEnd(void) { return tableEnd; }
   bool Check(uchar TableId, uchar Version, int SectionNumber);
   bool Processed(uchar TableId, uchar LastTableId, int SectionNumber, int LastSectionNumber, int SegmentLastSectionNumber = -1);
+       ///< Returns true if all sections of the table with the given TableId have been processed.
   bool Complete(void) { return complete; }
+       ///< Returns true if all sections of all tables have been processed.
   };
 
-class cSectionSyncerHash : public cHash<cEitTables> {
+class cEitTablesHash : public cHash<cEitTables> {
 public:
-  cSectionSyncerHash(void) : cHash(HASHSIZE, true) {};
+  cEitTablesHash(void) : cHash(HASHSIZE, true) {};
   };
 
 class cEitFilter : public cFilter {
 private:
   cMutex mutex;
-  cSectionSyncerHash sectionSyncerHash;
+  cEitTablesHash eitTablesHash;
   static time_t disableUntil;
 protected:
   virtual void Process(u_short Pid, u_char Tid, const u_char *Data, int Length);

epg.c

@@ -7,7 +7,7 @@
  * Original version (as used in VDR before 1.3.0) written by
  * Robert Schneider <Robert.Schneider@web.de> and Rolf Hakenes <hakenes@hippomi.de>.
  *
- * $Id: epg.c 5.1 2021/01/04 09:05:26 kls Exp $
+ * $Id: epg.c 5.2 2021/04/04 11:06:30 kls Exp $
  */
 
 #include "epg.h"
@@ -547,7 +547,7 @@ bool cEvent::Read(FILE *f, cSchedule *Schedule, int &Line)
                           unsigned int Version = 0xFF; // actual value is ignored
                           int n = sscanf(t, "%u %ld %d %X %X", &EventID, &StartTime, &Duration, &TableID, &Version);
                           if (n >= 3 && n <= 5) {
-                             Event = (cEvent *)Schedule->GetEvent(EventID, StartTime);
+                             Event = (cEvent *)Schedule->GetEventByTime(StartTime);
                              cEvent *newEvent = NULL;
                              if (Event)
                                 DELETENULL(Event->components);
@@ -913,6 +913,7 @@ cSchedule::cSchedule(tChannelID ChannelID)
   numTimers = 0;
   hasRunning = false;
   modified = 0;
+  onActualTp = false;
   presentSeen = 0;
 }
 
@@ -930,6 +931,13 @@ void cSchedule::DecNumTimers(void) const
   numTimersMutex.Unlock();
 }
 
+bool cSchedule::OnActualTp(uchar TableId)
+{
+  if ((TableId & 0xF0) == 0x50)
+     onActualTp = true;
+  return onActualTp;
+}
+
 cEvent *cSchedule::AddEvent(cEvent *Event)
 {
   events.Add(Event);
@@ -942,15 +950,21 @@ void cSchedule::DelEvent(cEvent *Event)
 {
   if (Event->schedule == this) {
      UnhashEvent(Event);
+     Event->schedule = NULL;
      events.Del(Event);
      }
 }
 
 void cSchedule::HashEvent(cEvent *Event)
 {
+  if (cEvent *p = eventsHashID.Get(Event->EventID()))
+     eventsHashID.Del(p, p->EventID());
   eventsHashID.Add(Event, Event->EventID());
-  if (Event->StartTime() > 0) // 'StartTime < 0' is apparently used with NVOD channels
+  if (Event->StartTime() > 0) { // 'StartTime < 0' is apparently used with NVOD channels
+     if (cEvent *p = eventsHashStartTime.Get(Event->StartTime()))
+        eventsHashStartTime.Del(p, p->StartTime());
      eventsHashStartTime.Add(Event, Event->StartTime());
+     }
 }
 
 void cSchedule::UnhashEvent(cEvent *Event)
@@ -990,6 +1004,7 @@ const cEvent *cSchedule::GetFollowingEvent(void) const
   return p;
 }
 
+#if DEPRECATED_SCHEDULE_GET_EVENT
 const cEvent *cSchedule::GetEvent(tEventID EventID, time_t StartTime) const
 {
   // Returns the event info with the given StartTime or, if no actual StartTime
@@ -999,6 +1014,19 @@ const cEvent *cSchedule::GetEvent(tEventID EventID, time_t StartTime) const
   else
      return eventsHashID.Get(EventID);
 }
+#endif
+
+const cEvent *cSchedule::GetEventById(tEventID EventID) const
+{
+  return eventsHashID.Get(EventID);
+}
+
+const cEvent *cSchedule::GetEventByTime(time_t StartTime) const
+{
+  if (StartTime > 0) // 'StartTime < 0' is apparently used with NVOD channels
+     return eventsHashStartTime.Get(StartTime);
+  return NULL;
+}
 
 const cEvent *cSchedule::GetEventAround(time_t Time) const
 {
@@ -1039,6 +1067,7 @@ void cSchedule::ClrRunningStatus(cChannel *Channel)
          if (p->RunningStatus() >= SI::RunningStatusPausing) {
             p->SetRunningStatus(SI::RunningStatusNotRunning, Channel);
             hasRunning = false;
+            SetModified();
             break;
             }
          }
@@ -1067,22 +1096,24 @@ void cSchedule::Sort(void)
 
 void cSchedule::DropOutdated(time_t SegmentStart, time_t SegmentEnd, uchar TableID, uchar Version)
 {
+  // Events are sorted by start time.
   if (SegmentStart > 0 && SegmentEnd > 0) {
      cEvent *p = events.First();
      while (p) {
            cEvent *n = events.Next(p);
-           if (p->EndTime() > SegmentStart) {
+           if (p->StartTime() >= SegmentStart) {
               if (p->StartTime() < SegmentEnd) {
-                 // The event overlaps with the given time segment.
+                 // The event starts within the given time segment.
-                 if (p->TableID() > TableID || p->TableID() == TableID && p->Version() != Version) {
+                 if ((p->TableID() > 0x4E || TableID == 0x4E) && (p->TableID() != TableID || p->Version() != Version)) {
-                    // The segment overwrites all events from tables with higher ids, and
+                    // The segment overwrites all events from tables with other ids, and
                     // within the same table id all events must have the same version.
+                    // Special consideration: table 0x4E can only be overwritten with the same id!
                     DelEvent(p);
                     }
                  }
+              }
            else
               break;
-              }
            p = n;
            }
      }

epg.h

@@ -7,7 +7,7 @@
  * Original version (as used in VDR before 1.3.0) written by
  * Robert Schneider <Robert.Schneider@web.de> and Rolf Hakenes <hakenes@hippomi.de>.
  *
- * $Id: epg.h 4.7 2017/05/28 12:59:20 kls Exp $
+ * $Id: epg.h 5.1 2021/04/04 11:06:30 kls Exp $
  */
 
 #ifndef __EPG_H
@@ -156,12 +156,14 @@ private:
   cHash<cEvent> eventsHashStartTime;
   mutable u_int16_t numTimers;// The number of timers that use this schedule
   bool hasRunning;
+  bool onActualTp;
   int modified;
   time_t presentSeen;
 public:
   cSchedule(tChannelID ChannelID);
   tChannelID ChannelID(void) const { return channelID; }
   bool Modified(int &State) const { bool Result = State != modified; State = modified; return Result; }
+  bool OnActualTp(uchar TableId);
   time_t PresentSeen(void) const { return presentSeen; }
   bool PresentSeenWithin(int Seconds) const { return time(NULL) - presentSeen < Seconds; }
   void SetModified(void) { modified++; }
@@ -183,7 +185,12 @@ public:
   const cList<cEvent> *Events(void) const { return &events; }
   const cEvent *GetPresentEvent(void) const;
   const cEvent *GetFollowingEvent(void) const;
+#define DEPRECATED_SCHEDULE_GET_EVENT 1
+#if DEPRECATED_SCHEDULE_GET_EVENT
   const cEvent *GetEvent(tEventID EventID, time_t StartTime = 0) const;
+#endif
+  const cEvent *GetEventById(tEventID EventID) const;
+  const cEvent *GetEventByTime(time_t StartTime) const;
   const cEvent *GetEventAround(time_t Time) const;
   void Dump(const cChannels *Channels, FILE *f, const char *Prefix = "", eDumpMode DumpMode = dmAll, time_t AtTime = 0) const;
   static bool Read(FILE *f, cSchedules *Schedules);