improved *.m3u import

This commit is contained in:
DYefremov
2021-01-31 16:27:35 +03:00
parent 7164c6debd
commit c4b6e009d5
4 changed files with 294 additions and 97 deletions

View File

@@ -3,9 +3,10 @@ import re
from enum import Enum
from urllib.parse import unquote, quote
from app.commons import log
from app.eparser.ecommons import BqServiceType, Service
from app.settings import SettingsType
from app.ui.uicommons import IPTV_ICON
from .ecommons import BqServiceType, Service
# url, description, urlkey, account, usrname, psw, s_type, iconsrc, iconsrc_b, group
NEUTRINO_FAV_ID_FORMAT = "{}::{}::{}::{}::{}::{}::{}::{}::{}::{}"
@@ -22,7 +23,7 @@ class StreamType(Enum):
E_SERVICE_HLS = "8739"
def parse_m3u(path, s_type, detect_encoding=True):
def parse_m3u(path, s_type, detect_encoding=True, params=None):
with open(path, "rb") as file:
data = file.read()
encoding = "utf-8"
@@ -37,28 +38,56 @@ def parse_m3u(path, s_type, detect_encoding=True):
encoding = enc.get("encoding", "utf-8")
aggr = [None] * 10
s_aggr = aggr[: -3]
services = []
groups = set()
counter = 0
marker_counter = 1
sid_counter = 1
name = None
picon = None
p_id = "1_0_1_0_0_0_0_0_0_0.png"
st = BqServiceType.IPTV.name
params = params or [0, 0, 0, 0]
for line in str(data, encoding=encoding, errors="ignore").splitlines():
if line.startswith("#EXTINF"):
name = line[1 + line.index(","):].strip()
inf, sep, line = line.partition(" ")
if not line:
line = inf
line, sep, name = line.rpartition(",")
data = re.split('"', line)
size = len(data)
if size < 3:
continue
d = {data[i].lower().strip(" ="): data[i + 1] for i in range(0, len(data) - 1, 2)}
picon = d.get("tvg-logo", None)
grp_name = d.get("group-title", None)
if grp_name not in groups:
groups.add(grp_name)
fav_id = MARKER_FORMAT.format(marker_counter, grp_name, grp_name)
marker_counter += 1
mr = Service(None, None, None, grp_name, *aggr[0:3], BqServiceType.MARKER.name, *aggr, fav_id, None)
services.append(mr)
elif line.startswith("#EXTGRP") and s_type is SettingsType.ENIGMA_2:
grp_name = line.strip("#EXTGRP:").strip()
if grp_name not in groups:
groups.add(grp_name)
fav_id = MARKER_FORMAT.format(counter, grp_name, grp_name)
counter += 1
fav_id = MARKER_FORMAT.format(marker_counter, grp_name, grp_name)
marker_counter += 1
mr = Service(None, None, None, grp_name, *aggr[0:3], BqServiceType.MARKER.name, *aggr, fav_id, None)
services.append(mr)
elif not line.startswith("#"):
url = line.strip()
fav_id = get_fav_id(url, name, s_type)
if name and url:
srv = Service(None, None, IPTV_ICON, name, *aggr[0:3], BqServiceType.IPTV.name, *aggr, fav_id, None)
params[0] = sid_counter
sid_counter += 1
fav_id = get_fav_id(url, name, s_type, params)
if all((name, url, fav_id)):
srv = Service(None, None, IPTV_ICON, name, *aggr[0:3], st, picon, p_id, *s_aggr, url, fav_id, None)
services.append(srv)
else:
log("*.m3u* parse error ['{}']: name[{}], url[{}], fav id[{}]".format(path, name, url, fav_id))
return services
@@ -86,12 +115,13 @@ def export_to_m3u(path, bouquet, s_type):
file.writelines(lines)
def get_fav_id(url, service_name, s_type):
def get_fav_id(url, service_name, settings_type, params=None, stream_type=None, s_type=1):
""" Returns fav id depending on the profile. """
if s_type is SettingsType.ENIGMA_2:
stream_type = StreamType.NONE_TS.value
return ENIGMA2_FAV_ID_FORMAT.format(stream_type, 1, 0, 0, 0, 0, quote(url), service_name, service_name, None)
elif s_type is SettingsType.NEUTRINO_MP:
if settings_type is SettingsType.ENIGMA_2:
stream_type = stream_type or StreamType.NONE_TS.value
params = params or (0, 0, 0, 0)
return ENIGMA2_FAV_ID_FORMAT.format(stream_type, s_type, *params, quote(url), service_name, service_name, None)
elif settings_type is SettingsType.NEUTRINO_MP:
return NEUTRINO_FAV_ID_FORMAT.format(url, "", 0, None, None, None, None, "", "", 1)

View File

@@ -3,7 +3,7 @@
The MIT License (MIT)
Copyright (c) 2018-2020 Dmitriy Yefremov
Copyright (c) 2018-2021 Dmitriy Yefremov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -31,7 +31,7 @@ Author: Dmitriy Yefremov
<!-- interface-license-type mit -->
<!-- interface-name DemonEditor -->
<!-- interface-description Enigma2 channel and satellites list editor for GNU/Linux. -->
<!-- interface-copyright 2018-2020 Dmitriy Yefremov -->
<!-- interface-copyright 2018-2021 Dmitriy Yefremov -->
<!-- interface-authors Dmitriy Yefremov -->
<object class="GtkDialog" id="search_unavailable_streams_dialog">
<property name="use-header-bar">1</property>
@@ -733,8 +733,8 @@ Author: Dmitriy Yefremov
<property name="gravity">center</property>
<signal name="response" handler="on_response" swapped="no"/>
<child type="action">
<object class="GtkButton" id="close_config_list_button">
<property name="label">gtk-close</property>
<object class="GtkButton" id="cancel_config_list_button">
<property name="label">gtk-cancel</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
@@ -742,17 +742,6 @@ Author: Dmitriy Yefremov
<property name="always_show_image">True</property>
</object>
</child>
<child type="action">
<object class="GtkButton" id="list_configuration_apply_button">
<property name="label">gtk-apply</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
<property name="always_show_image">True</property>
<signal name="clicked" handler="on_apply" swapped="no"/>
</object>
</child>
<child internal-child="vbox">
<object class="GtkBox" id="iptv_list_configuration_dialog_box">
<property name="can_focus">False</property>
@@ -774,7 +763,7 @@ Author: Dmitriy Yefremov
<property name="label_xalign">0</property>
<property name="shadow_type">in</property>
<child>
<object class="GtkBox">
<object class="GtkBox" id="reset_list_to_default_button">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="margin_left">5</property>
@@ -824,10 +813,12 @@ Author: Dmitriy Yefremov
</packing>
</child>
<child>
<object class="GtkSwitch" id="reset_to_default_lists_switch">
<object class="GtkButton" id="reset_list_to_default_butto">
<property name="label" translatable="yes">Reset to default</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<signal name="state-set" handler="on_reset_to_default" object="start_values_frame" swapped="no"/>
<property name="receives_default">True</property>
<signal name="clicked" handler="on_reset_to_default" swapped="no"/>
</object>
<packing>
<property name="expand">False</property>
@@ -836,20 +827,6 @@ Author: Dmitriy Yefremov
<property name="position">2</property>
</packing>
</child>
<child>
<object class="GtkLabel" id="reset_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="margin_right">2</property>
<property name="label" translatable="yes">Reset to default</property>
<property name="xalign">1</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">4</property>
</packing>
</child>
</object>
</child>
</object>
@@ -1218,7 +1195,7 @@ Author: Dmitriy Yefremov
</object>
</child>
<action-widgets>
<action-widget response="-6">close_config_list_button</action-widget>
<action-widget response="-6">cancel_config_list_button</action-widget>
</action-widgets>
</object>
<object class="GtkImage" id="remove_selection_image">

View File

@@ -5,17 +5,18 @@ from urllib.error import HTTPError
from urllib.parse import urlparse, unquote, quote
from urllib.request import Request, urlopen
from gi.repository import GLib
from gi.repository import GLib, Gio, GdkPixbuf
from app.commons import run_idle, run_task
from app.commons import run_idle, run_task, log
from app.eparser.ecommons import BqServiceType, Service
from app.eparser.iptv import NEUTRINO_FAV_ID_FORMAT, StreamType, ENIGMA2_FAV_ID_FORMAT, get_fav_id, MARKER_FORMAT
from app.eparser.iptv import (NEUTRINO_FAV_ID_FORMAT, StreamType, ENIGMA2_FAV_ID_FORMAT, get_fav_id, MARKER_FORMAT,
parse_m3u)
from app.settings import SettingsType
from app.tools.yt import YouTubeException, YouTube
from .dialogs import Action, show_dialog, DialogType, get_dialogs_string, get_message
from .main_helper import get_base_model, get_iptv_url, on_popup_menu
from .uicommons import (Gtk, Gdk, TEXT_DOMAIN, UI_RESOURCES_PATH, IPTV_ICON, Column, IS_GNOME_SESSION, KeyboardKey,
get_yt_icon)
from app.ui.dialogs import Action, show_dialog, DialogType, get_dialogs_string, get_message
from app.ui.main_helper import get_base_model, get_iptv_url, on_popup_menu
from app.ui.uicommons import (Gtk, Gdk, TEXT_DOMAIN, UI_RESOURCES_PATH, IPTV_ICON, Column, IS_GNOME_SESSION,
KeyboardKey, get_yt_icon)
_DIGIT_ENTRY_NAME = "digit-entry"
_ENIGMA2_REFERENCE = "{}:0:{}:{:X}:{:X}:{:X}:{:X}:0:0:0"
@@ -399,9 +400,10 @@ class SearchUnavailableDialog:
self._dialog.destroy()
class IptvListConfigurationDialog:
class IptvListDialog:
""" Base class for working with iptv lists. """
def __init__(self, transient, services, iptv_rows, bouquet, fav_model, s_type):
def __init__(self, transient, s_type):
handlers = {"on_apply": self.on_apply,
"on_response": self.on_response,
"on_stream_type_default_togged": self.on_stream_type_default_togged,
@@ -415,10 +417,6 @@ class IptvListConfigurationDialog:
"on_entry_changed": self.on_entry_changed,
"on_info_bar_close": self.on_info_bar_close}
self._rows = iptv_rows
self._services = services
self._bouquet = bouquet
self._fav_model = fav_model
self._s_type = s_type
builder = Gtk.Builder()
@@ -429,6 +427,8 @@ class IptvListConfigurationDialog:
self._dialog = builder.get_object("iptv_list_configuration_dialog")
self._dialog.set_transient_for(transient)
self._data_box = builder.get_object("iptv_list_data_box")
self._start_values_grid = builder.get_object("start_values_grid")
self._info_bar = builder.get_object("list_configuration_info_bar")
self._reference_label = builder.get_object("reference_label")
self._stream_type_check_button = builder.get_object("stream_type_default_check_button")
@@ -444,13 +444,15 @@ class IptvListConfigurationDialog:
self._list_nid_entry = builder.get_object("list_nid_entry")
self._list_namespace_entry = builder.get_object("list_namespace_entry")
self._reset_to_default_switch = builder.get_object("reset_to_default_lists_switch")
# style
self._style_provider = Gtk.CssProvider()
self._style_provider.load_from_path(UI_RESOURCES_PATH + "style.css")
# Style
style_provider = Gtk.CssProvider()
style_provider.load_from_path(UI_RESOURCES_PATH + "style.css")
self._default_elems = (self._stream_type_check_button, self._type_check_button, self._sid_auto_check_button,
self._tid_check_button, self._nid_check_button, self._namespace_check_button)
self._digit_elems = (self._list_srv_type_entry, self._list_sid_entry, self._list_tid_entry,
self._list_nid_entry, self._list_namespace_entry)
for el in self._digit_elems:
el.get_style_context().add_provider_for_screen(Gdk.Screen.get_default(), self._style_provider,
el.get_style_context().add_provider_for_screen(Gdk.Screen.get_default(), style_provider,
Gtk.STYLE_PROVIDER_PRIORITY_USER)
def show(self):
@@ -494,19 +496,58 @@ class IptvListConfigurationDialog:
self._list_namespace_entry.set_sensitive(not button.get_active())
@run_idle
def on_reset_to_default(self, item, active):
item.set_sensitive(not active)
def on_reset_to_default(self, item):
self._stream_type_combobox.set_active(1)
self._list_srv_type_entry.set_text("1")
for el in (self._list_sid_entry, self._list_nid_entry, self._list_tid_entry, self._list_namespace_entry):
for el in self._digit_elems[1:]:
el.set_text("0")
for el in (self._stream_type_check_button, self._type_check_button, self._sid_auto_check_button,
self._tid_check_button, self._nid_check_button, self._namespace_check_button):
for el in self._default_elems:
el.set_active(True)
def on_info_bar_close(self, bar=None, resp=None):
self._info_bar.set_visible(False)
def on_apply(self, item):
pass
@run_idle
def update_reference(self):
if is_data_correct(self._digit_elems):
stream_type = get_stream_type(self._stream_type_combobox)
self._reference_label.set_text(
_ENIGMA2_REFERENCE.format(stream_type, *[int(elem.get_text()) for elem in self._digit_elems]))
def on_entry_changed(self, entry):
if _PATTERN.search(entry.get_text()):
entry.set_name(_DIGIT_ENTRY_NAME)
else:
entry.set_name("GtkEntry")
self.update_reference()
def is_default_values(self):
return any(el.get_text() == "0" for el in self._digit_elems[2:])
def is_all_data_default(self):
return all(el.get_active() for el in self._default_elems)
class IptvListConfigurationDialog(IptvListDialog):
def __init__(self, transient, services, iptv_rows, bouquet, fav_model, s_type):
super().__init__(transient, s_type)
self._rows = iptv_rows
self._bouquet = bouquet
self._fav_model = fav_model
self._services = services
apply_button = Gtk.Button(visible=True,
image=Gtk.Image(icon_name="gtk-apply"),
always_show_image=True,
label=get_message("Apply"))
apply_button.connect("clicked", self.on_apply)
self._dialog.add_action_widget(apply_button, Gtk.ResponseType.APPLY)
@run_idle
def on_apply(self, item):
if not is_data_correct(self._digit_elems):
@@ -514,14 +555,13 @@ class IptvListConfigurationDialog:
return
if self._s_type is SettingsType.ENIGMA_2:
reset = self._reset_to_default_switch.get_active()
type_default = self._type_check_button.get_active()
tid_default = self._tid_check_button.get_active()
sid_auto = self._sid_auto_check_button.get_active()
nid_default = self._nid_check_button.get_active()
namespace_default = self._namespace_check_button.get_active()
stream_type = StreamType.NONE_TS.value if reset else get_stream_type(self._stream_type_combobox)
stream_type = get_stream_type(self._stream_type_combobox)
srv_type = "1" if type_default else self._list_srv_type_entry.get_text()
tid = "0" if tid_default else "{:X}".format(int(self._list_tid_entry.get_text()))
nid = "0" if nid_default else "{:X}".format(int(self._list_nid_entry.get_text()))
@@ -532,7 +572,7 @@ class IptvListConfigurationDialog:
data, sep, desc = fav_id.partition("http")
data = data.split(":")
if reset:
if self.is_all_data_default():
data[2], data[3], data[4], data[5], data[6] = "10000"
else:
data[0], data[2], data[4], data[5], data[6] = stream_type, srv_type, tid, nid, namespace
@@ -551,19 +591,178 @@ class IptvListConfigurationDialog:
self._info_bar.set_visible(True)
@run_idle
def update_reference(self):
if is_data_correct(self._digit_elems):
stream_type = get_stream_type(self._stream_type_combobox)
self._reference_label.set_text(
_ENIGMA2_REFERENCE.format(stream_type, *[int(elem.get_text()) for elem in self._digit_elems]))
def on_entry_changed(self, entry):
if _PATTERN.search(entry.get_text()):
entry.set_name(_DIGIT_ENTRY_NAME)
else:
entry.set_name("GtkEntry")
self.update_reference()
class M3uImportDialog(IptvListDialog):
""" Import dialog for *.m3u* playlists. """
def __init__(self, transient, s_type, path, callback):
super().__init__(transient, s_type)
self._callback = callback
self._services = None
self._url_count = 0
self._max_count = 0
self._is_download = False
self._cancellable = Gio.Cancellable()
self._dialog.set_title(get_message("Playlist import"))
# Progress
self._progress_bar = Gtk.ProgressBar(visible=False, valign="center")
self._spinner = Gtk.Spinner(active=False)
self._info_label = Gtk.Label(visible=True, ellipsize="end", max_width_chars=30)
load_label = Gtk.Label(label=get_message("Loading data..."))
self._spinner.bind_property("active", self._spinner, "visible")
self._spinner.bind_property("visible", load_label, "visible")
self._spinner.bind_property("active", self._start_values_grid, "sensitive", 4)
progress_box = Gtk.HBox(visible=True, spacing=2)
progress_box.add(self._progress_bar)
progress_box.pack_end(self._spinner, False, False, 0)
progress_box.pack_start(load_label, False, False, 0)
# Picons
self._picons_switch = Gtk.Switch(visible=True)
self._picon_box = Gtk.HBox(visible=False, sensitive=False, spacing=2)
self._picon_box.pack_end(self._picons_switch, False, False, 0)
self._picon_box.pack_end(Gtk.Label(visible=True, label=get_message("Download picons")), False, False, 0)
# Extra box
extra_box = Gtk.HBox(visible=True, spacing=2, margin_bottom=5, margin_top=5)
extra_box.set_center_widget(progress_box)
extra_box.pack_start(self._info_label, False, False, 5)
extra_box.pack_end(self._picon_box, True, True, 5)
frame = Gtk.Frame(visible=True)
frame.add(extra_box)
self._data_box.add(frame)
self._apply_button = Gtk.Button(visible=True,
image=Gtk.Image(icon_name="insert-link"),
always_show_image=True,
label=get_message("Import"))
self._apply_button.connect("clicked", self.on_apply)
self._dialog.add_action_widget(self._apply_button, Gtk.ResponseType.APPLY)
self._dialog.connect("delete-event", self.on_close)
self.get_m3u(path, s_type)
@run_task
def get_m3u(self, path, s_type):
try:
GLib.idle_add(self._spinner.set_property, "active", True)
self._services = parse_m3u(path, s_type)
for s in self._services:
if s.picon:
GLib.idle_add(self._picon_box.set_sensitive, True)
break
finally:
msg = "{} {}".format(get_message("Streams detected:"), len(self._services) if self._services else 0)
GLib.idle_add(self._info_label.set_text, msg)
GLib.idle_add(self._spinner.set_property, "active", False)
def on_apply(self, item):
if not is_data_correct(self._digit_elems):
show_dialog(DialogType.ERROR, self._dialog, "Error. Verify the data!")
return
picons = {}
services = self._services
if not self.is_all_data_default():
services = []
params = [int(el.get_text()) for el in self._digit_elems]
s_type = params[0]
params = params[1:]
stream_type = get_stream_type(self._stream_type_combobox)
for i, s in enumerate(self._services, start=params[0]):
# Skipping markers.
if not s.data_id:
services.append(s)
continue
params[0] = i
picon_id = "1_0_{:X}_{:X}_{:X}_{:X}_{:04X}0000_0_0_0.png".format(s_type, *params)
fav_id = get_fav_id(url=s.data_id,
service_name=s.service,
settings_type=self._s_type,
params=params,
stream_type=stream_type,
s_type=s_type)
picons[s.picon] = picon_id
services.append(s._replace(picon_id=picon_id, data_id=None, fav_id=fav_id))
if self._picons_switch.get_active():
if self.is_default_values():
show_dialog(DialogType.ERROR, self._dialog,
"Set values for TID, NID and Namespace for correct naming of the picons!")
return
self.download_picons(picons)
self._callback(services)
@run_task
def download_picons(self, picons):
self._is_download = True
GLib.idle_add(self._apply_button.set_sensitive, False)
GLib.idle_add(self._progress_bar.set_visible, True)
self._url_count = len(picons)
self._max_count = self._url_count
self._cancellable.reset()
for p in filter(None, picons):
if not self._is_download:
return
f = Gio.File.new_for_uri(p)
try:
GdkPixbuf.Pixbuf.new_from_stream_at_scale_async(f.read(cancellable=self._cancellable), 220, 132, False,
self._cancellable,
self.on_picon_load_done,
picons.get(p, None))
except GLib.GError as e:
self.update_progress()
if e.code != Gio.IOErrorEnum.CANCELLED:
log(str("Picon download error:{} {}").format(p, e))
def on_picon_load_done(self, file, result, user_data):
try:
pixbuf = GdkPixbuf.Pixbuf.new_from_stream_finish(result)
except GLib.GError as e:
if e.code != Gio.IOErrorEnum.CANCELLED:
log("Loading picon [{}] data error: {}".format(user_data, e))
finally:
self._info_label.set_text("Processing: {}".format(user_data))
self.update_progress()
def update_progress(self):
self._url_count -= 1
frac = 1 - self._url_count / self._max_count
self._progress_bar.set_fraction(frac)
if self._url_count == 0:
self._progress_bar.set_visible(False)
self._progress_bar.set_fraction(0.0)
self._apply_button.set_sensitive(True)
self._info_label.set_text(get_message("Done!"))
self._is_download = False
def on_response(self, dialog, response):
if response == Gtk.ResponseType.APPLY:
return True
if response == Gtk.ResponseType.CANCEL and not self._is_download or not self.on_close():
self._dialog.destroy()
def on_close(self, window=None, event=None):
if self._is_download:
if show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.OK:
self._is_download = False
self._cancellable.cancel()
return False
return True
return False
class YtListImportDialog:

View File

@@ -11,7 +11,7 @@ from gi.repository import GLib, Gio
from app.commons import run_idle, log, run_task, run_with_delay, init_logger
from app.connections import (HttpAPI, download_data, DownloadType, upload_data, test_http, TestException,
HttpApiException, STC_XML_FILE)
from app.eparser import get_blacklist, write_blacklist, parse_m3u
from app.eparser import get_blacklist, write_blacklist
from app.eparser import get_services, get_bouquets, write_bouquets, write_services, Bouquets, Bouquet, Service
from app.eparser.ecommons import CAS, Flag, BouquetService
from app.eparser.enigma.bouquets import BqServiceType
@@ -25,7 +25,7 @@ from .backup import BackupDialog, backup_data, clear_data_path
from .dialogs import show_dialog, DialogType, get_chooser_dialog, WaitDialog, get_message
from .download_dialog import DownloadDialog
from .imports import ImportDialog, import_bouquet
from .iptv import IptvDialog, SearchUnavailableDialog, IptvListConfigurationDialog, YtListImportDialog
from .iptv import IptvDialog, SearchUnavailableDialog, IptvListConfigurationDialog, YtListImportDialog, M3uImportDialog
from .main_helper import (insert_marker, move_items, rename, ViewTarget, set_flags, locate_in_services,
scroll_to, get_base_model, update_picons_data, copy_picon_reference, assign_picons,
remove_picon, is_only_one_item_selected, gen_bouquets, BqGenType, get_iptv_url, append_picons,
@@ -2153,17 +2153,8 @@ class Application(Gtk.Application):
self.show_error_dialog("No m3u file is selected!")
return
self._wait_dialog.show()
self.get_m3u(response)
@run_task
def get_m3u(self, path):
try:
channels = parse_m3u(path, self._s_type)
if channels and self._bq_selected:
GLib.idle_add(self.append_imported_services, channels)
finally:
GLib.idle_add(self._wait_dialog.hide)
if self._bq_selected:
M3uImportDialog(self._main_window, self._s_type, response, self.append_imported_services).show()
def append_imported_services(self, services):
bq_services = self._bouquets.get(self._bq_selected)