Channel/EPG scanning speedup and improved reporting

octoserve/var/channels/doepgscan.lua

@@ -68,6 +68,7 @@ local Max = 999999
 local Count = 0
 local ChannelCount = 0
 local EventCount = 0
+local ReportInterval = 1000
 
 Report(ChannelCount,"*")
 
@@ -82,7 +83,7 @@ else
    local Group
    for _,Group in ipairs(cl.GroupList) do
       for _,Channel in ipairs(Group.ChannelList) do
-         if Channel.ID then
+         if Channel.ID and not Channel.NoEPG then
             local Params = ""
             local p,v
             for p,v in Channel.Request:gmatch("(%a+)=([%w%.]+)") do
@@ -113,6 +114,7 @@ else
          end
       end
    end
+   cl = nil
 
    os.execute("rm "..outfile)
    local gzip = io.popen('gzip >'..outfile..".tmp","w")
@@ -121,22 +123,21 @@ else
       local newline = "\n "
       local Params
       local Key, t
+      local encode = newencoder()
       for Params,t in pairs(tl) do
          Report(Count,"of "..ChannelCount.." transponders, "..EventCount.." events")
-         Count = Count + 1
          Key = t.Key
          print("--------------------------------------------------------------")
-         collectgarbage();
          local cmd = 'octoscan --eit_sid '..t.Sids..' '..Options..Params..' '..ipAddr
          print(cmd)
          local octoscan = io.popen(cmd,"r")
          if octoscan then
+            tmp_count = ReportInterval
             while true do
                local line = octoscan:read("*l")
                if not line then
                   break
                end
---~                print(line)
                local eid = nil
                local name = nil
                local text = nil
@@ -149,7 +150,6 @@ else
                      if not line then
                         break
                      end
---~                      print(line)
                      if line == "END" then
                         if eid then
                            local event = { ID = Key..":"..eid }
@@ -169,13 +169,17 @@ else
                            --    event.Language = language
                            -- end
 
-                           local encode = newencoder()
                            local e = encode(event)
                            gzip:write(newline..e)
                            newline = ",\n "
                            EventCount = EventCount + 1
+
+                           tmp_count = tmp_count - 1
+                           if tmp_count == 0 then
+                              Report(Count,"of "..ChannelCount.." transponders, "..EventCount.." events")
+                              tmp_count = ReportInterval
+                           end
                         end
-                        collectgarbage();
                         break
                      end
 
@@ -198,6 +202,7 @@ else
             end
             octoscan:close()
          end
+         Count = Count + 1
       end
       gzip:write("\n]}")
       gzip:close()

octoserve/var/channels/doscan.lua

@@ -216,8 +216,9 @@ if tl.SourceList then
                   local sid = 0
                   local pids = ""
                   local tracks= { }
-                  local isradio = false
+                  local isradio = nil
                   local isencrypted = false
+                  local hasepg = nil
                   if line == "SERVICE" then
                      while true do
                         line = octoscan:read("*l")
@@ -275,7 +276,10 @@ if tl.SourceList then
                            local channel = { Title=sname,
                                              Request = '?'..Request..cireq.."&pids="..all_pids,
                                              Tracks=tracks,
-                                             ID=ID, Order=Order  }
+                                             ID=ID, Order=Order, Radio=isradio }
+                           if not hasepg then
+                              channel.NoEPG = true
+                           end
                            if ((not isradio) or (include_radio > 0)) and ((not isencrypted) or (include_encrypted > 0)) then
                               local group = GetGroup(ChannelList,gname)
                               if group then
@@ -289,6 +293,10 @@ if tl.SourceList then
                         local par,val = line:match("^ (%a+):(.*)")
                         if par == "RADIO" then
                            isradio = true
+                        elseif par == "EIT" then
+                           if val:sub(2,2) == "1" then
+                              hasepg = true
+                           end
                         elseif par == "PNAME" then
                            pname = val
                         elseif par == "SNAME" then

octoserve/var/www/channellists.html

@@ -16,6 +16,8 @@ var hasCable = false;
 var hasSat = false;
 var hasTer = false;
 
+var StartTime = null;
+
 var xmlhttp = new XMLHttpRequest();
 var url = "/channelscan.lua?select=keys";
 
@@ -156,8 +158,16 @@ function ScanStatus(response)
       }
       else if( s.status == "done" )
       {
+         var sec = "";
+         if( StartTime )
+         {
+            var t = new Date() - StartTime;
+            sec = ", time: " + Math.floor(t/1000) + " seconds";
+            StartTime = null;
+         }
+
          document.getElementById("scancount").firstChild.nodeValue = s.count;
-         document.getElementById("scantext").firstChild.nodeValue = s.msg;
+         document.getElementById("scantext").firstChild.nodeValue = s.msg + sec;
       }
       else if( s.status == "deleted" )
       {
@@ -235,6 +245,7 @@ function InitiateScan()
       ScanReq.send("select=scan" + param + "&sitables=1&sort=1&restartdms=1");
       document.getElementById("scancount").firstChild.nodeValue = "\u00A0";
       document.getElementById("scantext").firstChild.nodeValue = "Scanning...";
+      StartTime = new Date();
    }
    else
    {
@@ -246,11 +257,12 @@ function InitiateScan()
 function InitiateEPGScan()
 {
    DisableButtons(true);
-      ScanReq.open("POST", "/channelscan.lua", true);
+   ScanReq.open("POST", "/channelscan.lua", true);
-      ScanReq.setRequestHeader("Content-type","application/x-www-form-urlencoded");
+   ScanReq.setRequestHeader("Content-type","application/x-www-form-urlencoded");
-      ScanReq.send("select=epg");
+   ScanReq.send("select=epg");
-      document.getElementById("scancount").firstChild.nodeValue = "\u00A0";
+   document.getElementById("scancount").firstChild.nodeValue = "\u00A0";
-      document.getElementById("scantext").firstChild.nodeValue = "Scanning...";
+   document.getElementById("scantext").firstChild.nodeValue = "Scanning...";
+   StartTime = new Date();
 }
 
 function PollStatus()

octoserve/var/www/epg.html

@@ -107,7 +107,8 @@ function LoadEPG()
 function ChannelListResponse(response)
 {
    ChannelList = JSON.parse(response);
-   if( ChannelList.GroupList[0].ChannelList[0].ID )
+   var Valid = false;
+   if( ChannelList.GroupList[0].ChannelList )
    {
       ChannelLookup = new Object();
       for(var i = 0; i < ChannelList.GroupList.length; i += 1)
@@ -115,9 +116,16 @@ function ChannelListResponse(response)
          var Group = ChannelList.GroupList[i];
          for(var j = 0; j < Group.ChannelList.length; j += 1)
          {
-            ChannelLookup[Group.ChannelList[j].ID] = Group.ChannelList[j];
+            if( Group.ChannelList[j].ID )
+            {
+               Valid = true;
+               ChannelLookup[Group.ChannelList[j].ID] = Group.ChannelList[j];
+            }
          }
       }
+   }
+   if( Valid )
+   {
       LoadEPG();
    }
    else