#!/usr/bin/env python # -*- coding: utf-8 -*- ################################################################################ # # Copyright (C) 2013-present Neil MacLeod (texturecache@nmacleod.com) # # This Program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # # This Program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # Simple utility to query, validate, clean and refresh the Kodi texture cache. # # https://github.com/MilhouseVH/texturecache.py # # Usage: # # See built-in help (run script without parameters), or the README file # on Github for more details. # ################################################################################ import os, sys, platform, re, datetime, time import socket, base64, hashlib import threading, random import errno, codecs import subprocess import tempfile try: import json except: import simplejson as json if sys.version_info >= (3, 0): import configparser as ConfigParser import io as StringIO import http.client as httplib import urllib.request as urllib2 import queue as Queue basestring = (str, bytes) else: import ConfigParser, StringIO, httplib, urllib2, Queue lock = threading.RLock() # # Config class. Will be a global object. # class MyConfiguration(object): def __init__(self, argv): self.VERSION = "2.5.4" self.GITHUB = "https://raw.github.com/MilhouseVH/texturecache.py/master" self.ANALYTICS_GOOD = "http://goo.gl/BjH6Lj" self.DEBUG = True if os.environ.get("PYTHONDEBUG", "n").lower()=="y" else False self.GLOBAL_SECTION = "global" self.THIS_SECTION = self.GLOBAL_SECTION self.CONFIG_NAME = "texturecache.cfg" self.HAS_PVR = False # https://github.com/xbmc/xbmc/blob/master/xbmc/settings/AdvancedSettings.cpp m_pictureExtensions = ".png|.jpg|.jpeg|.bmp|.gif|.ico|.tif|.tiff|.tga|.pcx|.cbz|.zip|.cbr|.rar|.dng|.nef|.cr2|.crw|.orf|.arw|.erf|.3fr|.dcr|.x3f|.mef|.raf|.mrw|.pef|.sr2|.rss" m_musicExtensions = ".nsv|.m4a|.flac|.aac|.strm|.pls|.rm|.rma|.mpa|.wav|.wma|.ogg|.mp3|.mp2|.m3u|.mod|.amf|.669|.dmf|.dsm|.far|.gdm|.imf|.it|.m15|.med|.okt|.s3m|.stm|.sfx|.ult|.uni|.xm|.sid|.ac3|.dts|.cue|.aif|.aiff|.wpl|.ape|.mac|.mpc|.mp+|.mpp|.shn|.zip|.rar|.wv|.nsf|.spc|.gym|.adx|.dsp|.adp|.ymf|.ast|.afc|.hps|.xsp|.xwav|.waa|.wvs|.wam|.gcm|.idsp|.mpdsp|.mss|.spt|.rsd|.mid|.kar|.sap|.cmc|.cmr|.dmc|.mpt|.mpd|.rmt|.tmc|.tm8|.tm2|.oga|.url|.pxml|.tta|.rss|.cm3|.cms|.dlt|.brstm|.wtv|.mka|.tak|.opus|.dff|.dsf" m_videoExtensions = ".m4v|.3g2|.3gp|.nsv|.tp|.ts|.ty|.strm|.pls|.rm|.rmvb|.m3u|.m3u8|.ifo|.mov|.qt|.divx|.xvid|.bivx|.vob|.nrg|.img|.iso|.pva|.wmv|.asf|.asx|.ogm|.m2v|.avi|.bin|.dat|.mpg|.mpeg|.mp4|.mkv|.mk3d|.avc|.vp3|.svq3|.nuv|.viv|.dv|.fli|.flv|.rar|.001|.wpl|.zip|.vdr|.dvr-ms|.xsp|.mts|.m2t|.m2ts|.evo|.ogv|.sdp|.avs|.rec|.url|.pxml|.vc1|.h264|.rcv|.rss|.mpls|.webm|.bdmv|.wtv" m_subtitlesExtensions = ".utf|.utf8|.utf-8|.sub|.srt|.smi|.rt|.txt|.ssa|.text|.ssa|.aqt|.jss|.ass|.idx|.ifo|.rar|.zip" # These features become available with the respective API version self.JSON_VER_CAPABILITIES = {"setresume": (6, 2, 0), "profilesupport": (6, 6, 0), "texturedb": (6, 9, 0), "removeart": (6, 9, 1), "setseason": (6, 10, 0), "setmovieset": (6, 12, 0), "setsettings": (6, 13, 0), "filternullval": (6, 13, 1), "isodates": (6, 13, 2), "debugextralog": (6, 15, 3), "dpmsnotify": (6, 16, 0), "openplayercoredef":(6, 18, 3), "libshowdialogs": (6, 19, 0), "exitcode": (6, 21, 0), "refreshrefactor": (6, 27, 0), "votesnogrouping": (7, 1, 0), "playerprocessinfo":(7, 20, 0), "codecinforemoved": (7, 21, 0), "musichasart": (9, 4, 2), "profiledirectory": (999, 99, 9) } self.SetJSONVersion(0, 0, 0) namedSection = False serial_urls = "assets\.fanart\.tv" embedded_urls = "^video, ^music, ^DefaultVideo.png" if MyUtility.isPython3: config = ConfigParser.ConfigParser(strict=False) else: config = ConfigParser.SafeConfigParser() self.config = config #Use @section argument if passed on command line #Use list(argv) so that a copy of argv is iterated over, making argv.remove() safe to use. for arg in list(argv): if re.search("^[ ]*@section[ ]*=", arg): self.THIS_SECTION = arg.split("=", 1)[1].strip() namedSection = True argv.remove(arg) #Use @config if passed on command line for arg in list(argv): if re.search("^[ ]*@config[ ]*=", arg): self.CONFIG_NAME = arg.split("=", 1)[1].strip() argv.remove(arg) # Use the default or user specified config filename. # If it doesn't exist and is a relative filename, try and find a config # file in the current directory, otherwise look in same directory as script itself. self.FILENAME = os.path.expanduser(self.CONFIG_NAME) if not os.path.exists(self.FILENAME) and not os.path.isabs(self.FILENAME): self.FILENAME = "%s%s%s" % (os.getcwd(), os.sep, self.CONFIG_NAME) if not os.path.exists(self.FILENAME): self.FILENAME = "%s%s%s" % (os.path.dirname(os.path.abspath(__file__)), os.sep, self.CONFIG_NAME) if not os.path.exists(self.FILENAME): self.FILENAME = os.path.expanduser("~/.config/%s" % self.CONFIG_NAME) cfg = StringIO.StringIO() cfg.write("[%s]\n" % self.GLOBAL_SECTION) if os.path.exists(self.FILENAME): cfg.write(open(self.FILENAME, "r").read()) cfg.write("\n") cfg.seek(0, os.SEEK_SET) if MyUtility.isPython3: config.read_file(cfg) else: config.readfp(cfg) # If a specific section is not passed on the command line, check the config # to see if there is a default section property, and if not then default to # the global default section. if not namedSection: self.THIS_SECTION = self.getValue(config, "section", self.GLOBAL_SECTION) # If the section is not present, bail if not config.has_section(self.THIS_SECTION): print("Section [%s] is not a valid section in this config file" % self.THIS_SECTION) sys.exit(2) #Add any command line settings - eg. @kodi.host=192.168.0.8 - to the named section. for arg in list(argv): arg_match = re.match("^[ ]*@([^ ]+)[ ]*=(.*)", arg) if arg_match and len(arg_match.groups()) == 2: argKey, argVal = arg.split("=", 1) config.set(self.THIS_SECTION, argKey.strip()[1:], argVal.strip()) argv.remove(arg) if not self.DEBUG and self.getBoolean(config, "debug", ""): self.DEBUG = self.getBoolean(config, "debug", "no") self.IDFORMAT = self.getValue(config, "format", "%06d") self.FSEP = self.getValue(config, "sep", "|") UD_SYS_DEFAULT = self.getdefaultuserdata("Kodi") if UD_SYS_DEFAULT is None: UD_SYS_DEFAULT = self.getdefaultuserdata("XBMC") if UD_SYS_DEFAULT is None: UD_SYS_DEFAULT = "~/userdata" self.KODI_BASE = os.path.expanduser(self.getValue(config, "userdata", UD_SYS_DEFAULT)) self.TEXTUREDB = self.getValue(config, "dbfile", "Database/Textures13.db") self.THUMBNAILS = self.getValue(config, "thumbnails", "Thumbnails") self.CURRENT_PROFILE = {"label": "", "lockmode": 0, "thumbnail": "", "directory": "", "tc.profilepath": self.KODI_BASE} # Not yet known self.PROFILE_ENABLED = self.getBoolean(config, "profile.enabled", "yes") self.PROFILE_MASTER = self.getValue(config, "profile.master", "Master user") self.PROFILE_AUTOLOAD = self.getBoolean(config, "profile.autoload", "yes") self.PROFILE_RETRY = int(self.getValue(config, "profile.retry", "60")) self.PROFILE_WAIT = int(self.getValue(config, "profile.wait", "0")) self.PROFILE_NAME = self.getValue(config, "profile.name", self.PROFILE_MASTER) self.PROFILE_PASSWORD = self.getValue(config, "profile.password", "") self.PROFILE_ENCRYPTED = self.getBoolean(config, "profile.password.encrypted", "no") self.PROFILE_DIRECTORY = self.getValue(config, "profile.directory", "") if self.PROFILE_DIRECTORY == "" and self.PROFILE_NAME != self.PROFILE_MASTER: self.PROFILE_DIRECTORY = self.PROFILE_NAME if self.PROFILE_DIRECTORY != "": self.PROFILE_DIRECTORY = os.path.join("profiles", self.PROFILE_DIRECTORY) # Read library and textures data in chunks to minimise server/client memory usage self.CHUNKED = self.getBoolean(config, "chunked", "yes") self.DBJSON = self.getValue(config, "dbjson", "auto") self.USEJSONDB = self.getBoolean(config, "dbjson", "yes") if self.KODI_BASE[-1:] not in ["/", "\\"]: self.KODI_BASE += "/" if self.THUMBNAILS[-1:] not in ["/", "\\"]: self.THUMBNAILS += "/" self.KODI_BASE = self.KODI_BASE.replace("/", os.sep) self.TEXTUREDB = self.TEXTUREDB.replace("/", os.sep) self.THUMBNAILS = self.THUMBNAILS.replace("/", os.sep) self.HAS_THUMBNAILS_FS = os.path.exists(self.getFilePath()) self.KODI_HOST = self.getValue(config, "xbmc.host", None, True) if self.KODI_HOST is None: self.KODI_HOST = self.getValue(config, "kodi.host", "localhost") self.WEB_PORT = self.getValue(config, "webserver.port", "8080") self.WEB_SINGLESHOT = self.getBoolean(config, "webserver.singleshot", "no") self.RPC_PORT = self.getValue(config, "rpc.port", "9090") self.RPC_IPVERSION = self.getValue(config, "rpc.ipversion", "") self.RPC_RETRY = int(self.getValue(config, "rpc.retry", "12")) self.RPC_RETRY = 0 if self.RPC_RETRY < 0 else self.RPC_RETRY web_user = self.getValue(config, "webserver.username", "") web_pass = self.getValue(config, "webserver.password", "") self.WEB_CONNECTTIMEOUT = self.getValue(config, "webserver.ctimeout", 0.5, allowundefined=True) if self.WEB_CONNECTTIMEOUT: self.WEB_CONNECTTIMEOUT = float(self.WEB_CONNECTTIMEOUT) self.RPC_CONNECTTIMEOUT = self.getValue(config, "rpc.ctimeout", 0.5, allowundefined=True) if self.RPC_CONNECTTIMEOUT: self.RPC_CONNECTTIMEOUT = float(self.RPC_CONNECTTIMEOUT) if (web_user and web_pass): token = "%s:%s" % (web_user, web_pass) if MyUtility.isPython3: self.WEB_AUTH_TOKEN = base64.encodebytes(bytes(token, "utf-8")).decode() else: self.WEB_AUTH_TOKEN = base64.encodestring(token) self.WEB_AUTH_TOKEN = self.WEB_AUTH_TOKEN.replace("\n", "") else: self.WEB_AUTH_TOKEN = None self.QUERY_SEASONS = self.getBoolean(config, "query.seasons", "yes") self.QUERY_EPISODES = self.getBoolean(config, "query.episodes", "yes") if self.QUERY_SEASONS else False self.DOWNLOAD_THREADS_DEFAULT = int(self.getValue(config, "download.threads", "2")) self.DOWNLOAD_RETRY = int(self.getValue(config, "download.retry", "3")) self.DOWNLOAD_PRIME = self.getBoolean(config, "download.prime", "yes") # It seems that Files.Preparedownload is sufficient to populate the texture cache # so there is no need to actually download the artwork. # v0.8.8: Leave enabled for now, may only be sufficient in recent builds. self.DOWNLOAD_PAYLOAD = self.getBoolean(config, "download.payload","yes") self.DOWNLOAD_THREADS = {} for x in ["addons", "albums", "artists", "songs", "movies", "sets", "tags", "tvshows", "musicvideos", "pvr.tv", "pvr.radio"]: temp = int(self.getValue(config, "download.threads.%s" % x, self.DOWNLOAD_THREADS_DEFAULT)) self.DOWNLOAD_THREADS["download.threads.%s" % x] = temp self.SINGLETHREAD_URLS = self.getPatternFromList(config, "singlethread.urls", serial_urls, allowundefined=True) self.XTRAJSON = {} self.QA_FIELDS = {} # Modifiers - all are optional: # ? - warn instead of fail when item is missing (or present in fail/warn pattern) # # - don't warn when missing, warn otherwise (unless art present in fail list, then fail) # ! - don't warn when missing, fail otherwise (unless art present in warn list, then warn) self.QA_FIELDS["qa.art.addons"] = "thumbnail" self.QA_FIELDS["qa.art.agenres"] = "thumbnail" self.QA_FIELDS["qa.art.vgenres"] = "thumbnail" self.QA_FIELDS["qa.art.artists"] = "fanart, thumbnail" self.QA_FIELDS["qa.art.albums"] = "fanart, thumbnail" self.QA_FIELDS["qa.art.songs"] = "fanart, thumbnail" self.QA_FIELDS["qa.blank.movies"] = "plot, mpaa" self.QA_FIELDS["qa.art.movies"] = "fanart, poster" self.QA_FIELDS["qa.art.sets"] = "fanart, poster" self.QA_FIELDS["qa.blank.tvshows.tvshow"] = "plot" self.QA_FIELDS["qa.blank.tvshows.episode"] = "plot" self.QA_FIELDS["qa.art.tvshows.tvshow"] = "fanart, banner, poster" self.QA_FIELDS["qa.art.tvshows.season"] = "poster" self.QA_FIELDS["qa.art.tvshows.episode"] = "thumb" self.QA_FIELDS["qa.art.pvr.tv.channel"] = "thumbnail" self.QA_FIELDS["qa.art.pvr.radio.channel"] = "thumbnail" for x in ["addons", "albums", "artists", "songs", "musicvideos", "movies", "sets", "tvshows.tvshow", "tvshows.season", "tvshows.episode", "pvr.tv", "pvr.radio", "pvr.tv.channel", "pvr.radio.channel", "agenres", "vgenres"]: key = "extrajson.%s" % x temp = self.getValue(config, key, "") self.XTRAJSON[key] = temp if temp != "" else None for f in ["zero", "blank", "art"]: key = "qa.%s.%s" % (f, x) try: temp = self.getValue(config, key, None) except: temp = None if temp and temp.startswith("+"): temp = temp[1:] temp2 = self.QA_FIELDS.get(key, "") if temp2 != "": temp2 = "%s, " % temp2 temp = "%s%s " % (temp2, temp.strip()) self.QA_FIELDS[key] = temp else: self.QA_FIELDS[key] = temp if temp is not None else self.QA_FIELDS.get(key, None) self.QAPERIOD = int(self.getValue(config, "qaperiod", "30")) adate = datetime.date.today() - datetime.timedelta(days=self.QAPERIOD) self.QADATE = adate.strftime("%Y-%m-%d") if self.QAPERIOD >= 0 else None self.QA_FILE = self.getBoolean(config, "qa.file", "no") self.QA_FAIL_CHECKEXISTS = self.getBoolean(config, "qa.fail.checkexists", "yes") self.QA_FAIL_MISSING_LOCAL_ART = self.getBoolean(config, "qa.fail.missinglocalart", "no") self.QA_FAIL_TYPES = self.getPatternFromList(config, "qa.fail.urls", embedded_urls, allowundefined=True) self.QA_WARN_TYPES = self.getPatternFromList(config, "qa.warn.urls", "") (self.QA_NFO_REFRESH, self.qa_nfo_refresh_date, self.qa_nfo_refresh_date_fmt) = self.getRelativeDateAndFormat(config, "qa.nfo.refresh", "") self.QA_USEOLDREFRESHMETHOD = self.getBoolean(config, "qa.useoldrefreshmethod", "yes") self.CACHE_CAST_THUMB = self.getBoolean(config, "cache.castthumb", "no") (self.CACHE_REFRESH, self.cache_refresh_date, self.cache_refresh_date_fmt) = self.getRelativeDateAndFormat(config, "cache.refresh", "") yn = "yes" if self.getBoolean(config, "cache.extra", "no") else "no" self.CACHE_EXTRA_FANART = self.getBoolean(config, "cache.extrafanart", yn) self.CACHE_EXTRA_THUMBS = self.getBoolean(config, "cache.extrathumbs", yn) # http://kodi.wiki/view/Add-on:VideoExtras self.CACHE_VIDEO_EXTRAS = self.getBoolean(config, "cache.videoextras", yn) self.CACHE_EXTRA = (self.CACHE_EXTRA_FANART or self.CACHE_EXTRA_THUMBS or self.CACHE_VIDEO_EXTRAS) self.LOGFILE = self.getValue(config, "logfile", "") self.LOGVERBOSE = self.getBoolean(config, "logfile.verbose", "yes") self.LOGUNIQUE = self.getBoolean(config, "logfile.unique", "no") self.LOGDCACHE = self.getBoolean(config, "logfile.dcache", "no") self.CACHE_ARTWORK = self.getSimpleList(config, "cache.artwork", "") self.CACHE_IGNORE_TYPES = self.getPatternFromList(config, "cache.ignore.types", embedded_urls, allowundefined=True) self.PRUNE_RETAIN_TYPES = self.getPatternFromList(config, "prune.retain.types", "") self.CACHE_DROP_INVALID_FILE = self.getValue(config, "cache.dropfile", "") # Fix patterns as we now strip image:// from the URLs, so we need to remove # this prefix from any legacy patterns that may be specified by the user for index, r in enumerate(self.CACHE_IGNORE_TYPES): self.CACHE_IGNORE_TYPES[index] = re.compile(re.sub("^\^image://", "^", r.pattern)) for index, r in enumerate(self.PRUNE_RETAIN_TYPES): self.PRUNE_RETAIN_TYPES[index] = re.compile(re.sub("^\^image://", "^", r.pattern)) self.PRUNE_RETAIN_PREVIEWS = self.getBoolean(config, "prune.retain.previews", "yes") self.PRUNE_RETAIN_PICTURES = self.getBoolean(config, "prune.retain.pictures", "no") self.PRUNE_RETAIN_CHAPTERS = self.getBoolean(config, "prune.retain.chapters", "yes") self.MISSING_IGNORE_PATTERNS = self.getPatternFromList(config, "missing.ignore.patterns", "", allowundefined=True) self.picture_filetypes = m_pictureExtensions.split("|") self.PICTURE_FILETYPES_EX = self.getFileExtList(config, "picture.filetypes", "") self.picture_filetypes.extend(self.PICTURE_FILETYPES_EX) self.video_filetypes = m_videoExtensions.split("|") self.VIDEO_FILETYPES_EX = self.getFileExtList(config, "video.filetypes", "") self.video_filetypes.extend(self.VIDEO_FILETYPES_EX) self.audio_filetypes = m_musicExtensions.split("|") self.AUDIO_FILETYPES_EX = self.getFileExtList(config, "audio.filetypes", "") self.audio_filetypes.extend(self.AUDIO_FILETYPES_EX) self.subtitle_filetypes = m_subtitlesExtensions.split("|") self.SUBTITLE_FILETYPES_EX = self.getFileExtList(config, "subtitle.filetypes", "") self.subtitle_filetypes.extend(self.SUBTITLE_FILETYPES_EX) self.IGNORE_PLAYLISTS = self.getBoolean(config, "ignore.playlists","yes") self.RECACHEALL = self.getBoolean(config, "allow.recacheall","no") self.CHECKUPDATE = self.getBoolean(config, "checkupdate", "yes") self.AUTOUPDATE = self.getBoolean(config, "autoupdate", "yes") self.LASTRUNFILE = self.getValue(config, "lastrunfile", "") self.LASTRUNFILE_DATETIME = None if self.LASTRUNFILE and os.path.exists(self.LASTRUNFILE): temp = datetime.datetime.fromtimestamp(os.path.getmtime(self.LASTRUNFILE)) self.LASTRUNFILE_DATETIME = temp.strftime("%Y-%m-%d %H:%M:%S") self.ORPHAN_LIMIT_CHECK = self.getBoolean(config, "orphan.limit.check", "yes") self.CACHE_HIDEALLITEMS = self.getBoolean(config, "cache.hideallitems", "no") self.WATCHEDOVERWRITE = self.getBoolean(config, "watched.overwrite", "no") self.MAC_ADDRESS = self.getValue(config, "network.mac", "") self.ADD_SET_MEMBERS = self.getBoolean(config, "setmembers", "yes") self.ADD_SONG_MEMBERS = self.getBoolean(config, "songmembers", "no") self.PURGE_MIN_LEN = int(self.getValue(config, "purge.minlen", "5")) self.OMDB_API_KEY = self.getValue(config, "omdb.apikey", None, True) self.IMDB_FIELDS_MOVIES = self.getExRepList(config, "imdb.fields.movies", ["rating", "votes", "top250"], True) self.IMDB_FIELDS_TVSHOWS = self.getExRepList(config, "imdb.fields.tvshows", ["rating", "votes"], True) self.IMDB_IGNORE_TVTITLES = self.getSimpleList(config, "imdb.ignore.tvtitles", "", True, "|") self.IMDB_MAP_TVTITLES = self.getSimpleList(config, "imdb.map.tvtitles", "", True, "|") self.IMDB_TRANSLATE_TVTITLES = self.getSimpleList(config, "imdb.translate.tvtitles", "", True, "|", False) self.IMDB_TRANSLATE_TVYEARS = self.getSimpleList(config, "imdb.translate.tvyears", "", True, "|") self.IMDB_IGNORE_MISSING_EPISODES = self.getBoolean(config, "imdb.ignore.missing.episodes", "no") self.IMDB_THREADS = int(self.getValue(config, "imdb.threads", 10)) self.IMDB_THREADS = 1 if self.IMDB_THREADS < 1 else self.IMDB_THREADS self.IMDB_THREADS = 20 if self.IMDB_THREADS > 20 else self.IMDB_THREADS self.IMDB_TIMEOUT = self.getValue(config, "imdb.timeout", 15.0, allowundefined=True) if self.IMDB_TIMEOUT: self.IMDB_TIMEOUT = float(self.IMDB_TIMEOUT) self.IMDB_RETRY = int(self.getValue(config, "imdb.retry", 3)) self.IMDB_RETRY = 0 if self.IMDB_RETRY < 0 else self.IMDB_RETRY self.IMDB_GROUPING = self.getValue(config, "imdb.grouping", ",", allowundefined=True) self.IMDB_GROUPING = self.IMDB_GROUPING if self.IMDB_GROUPING is not None else "" self.IMDB_DEL_PARENTHESIS = self.getBoolean(config, "imdb.delete_parenthesis", "yes") self.IMDB_PERIOD = self.getValue(config, "imdb.period", None, allowundefined=True) if self.IMDB_PERIOD: adate = datetime.date.today() - datetime.timedelta(days=int(self.IMDB_PERIOD)) self.IMDB_PERIOD_FROM = adate.strftime("%Y-%m-%d") else: self.IMDB_PERIOD_FROM = None self.BIN_TVSERVICE = self.getValue(config, "bin.tvservice", "/usr/bin/tvservice") self.BIN_VCGENCMD = self.getValue(config, "bin.vcgencmd", "/usr/bin/vcgencmd", allowundefined=True) self.BIN_CECCONTROL = self.getValue(config, "bin.ceccontrol", "") self.HDMI_FORCE_HOTPLUG = self.getBoolean(config, "hdmi.force.hotplug", "no") self.HDMI_IGNORE_SUSPEND = self.getBoolean(config, "hdmi.ignoresuspend", "no") self.HDMI_IGNORE_DISABLE = self.getBoolean(config, "hdmi.ignoredisable", "no") self.HDMI_IGNORE_PLAYER = self.getBoolean(config, "hdmi.ignoreplayer", "no") self.HDMI_IGNORE_LIBRARY = self.getBoolean(config, "hdmi.ignorelibrary", "no") # Use a smaller cache on ARM systems, based on the assumption that ARM systems # will have less memory than other platforms defSize = "512" if platform.machine().lower().startswith("arm") else "2048" self.DCACHE_SIZE = int(self.getValue(config, "dcache.size", defSize)) self.DCACHE_AGELIMIT = int(self.getValue(config, "dcache.agelimit", "180")) self.FILTER_FIELD = self.getValue(config, "filter", "") self.FILTER_OPERATOR = self.getValue(config, "filter.operator", "contains") self.SEARCH_ENCODE = self.getBoolean(config, "encode", "yes") self.POSTER_WIDTH = int(self.getValue(config, "posterwidth", "5")) self.CLEAN_SHOW_DIALOGS = self.getBoolean(config, "clean.showdialogs", "no") self.SCAN_SHOW_DIALOGS = self.getBoolean(config, "scan.showdialogs", "no") self.LOG_REPLAY_FILENAME = self.getValue(config, "replayfile", "") self.log_replay_fmap = {} self.log_replay_tmap = {} def getdefaultuserdata(self, appid): atv2_path = "/User/Library/Preferences/%s/userdata" % appid macosx_path = "~/Library/Application Support/%s/userdata" % appid linux1_path = "/var/lib/%s/.%s/userdata" % (appid.lower(), appid.lower()) linux2_path = "~/.%s/userdata" % appid.lower() android1_path = "Android/data/org.%s.%s/files/.%s/userdata" % (appid.lower(), appid.lower(), appid.lower()) android2_path = "/sdcard/%s" % android1_path firetv_path = "/storage/emulated/0/%s" % android1_path if sys.platform == "win32": win32_path = "%s\\%s\\userdata" % (os.environ["appdata"], appid) if os.path.exists(win32_path): return win32_path elif sys.platform == "darwin" and os.path.exists(atv2_path): return atv2_path elif sys.platform == "darwin" and os.path.exists(os.path.expanduser(macosx_path)): return macosx_path else: #Linux/Android if os.path.exists(os.path.expanduser(linux1_path)): return linux1_path elif os.path.exists(os.path.expanduser(linux2_path)): return linux2_path elif os.path.exists(firetv_path): return firetv_path elif os.path.exists(android2_path): return android2_path elif os.path.exists(android1_path): return android1_path return None def SetJSONVersion(self, major, minor, patch): self.JSON_VER = (major, minor, patch) self.JSON_VER_STR = "v%d.%d.%d" % (major, minor, patch) # Allow restoration of resume points with Gotham+ self.JSON_HAS_SETRESUME = self.HasJSONCapability("setresume") # Allow null artwork items in set self.JSON_HAS_SETNULL = self.HasJSONCapability("removeart") # JSON API supports VideoLibrary.SetSeasonDetails and VideoLibrary.SetMovieSetDetails self.JSON_HAS_SETSEASON = self.HasJSONCapability("setseason") self.JSON_HAS_SETMOVIESET = self.HasJSONCapability("setmovieset") # JSON API has support for Textures database access self.JSON_HAS_TEXTUREDB = self.HasJSONCapability("texturedb") # Filter is/isnot operator broken for empty string value (""), use only if API is fixed self.JSON_HAS_FILTERNULLVALUE = self.HasJSONCapability("filternullval") # https://github.com/xbmc/xbmc/commit/20717c1b0cc2e5b35996be52cabd7267e0799995 self.JSON_HAS_ISO_DATES = self.HasJSONCapability("isodates") # https://github.com/xbmc/xbmc/pull/4766 self.JSON_HAS_DPMS_NOTIFY = self.HasJSONCapability("dpmsnotify") # https://github.com/xbmc/xbmc/commit/77812fbc7e35aaea67e5df31a96a932f85595184 self.JSON_HAS_DEBUG_EXTRA_LOG = self.HasJSONCapability("debugextralog") # https://github.com/xbmc/xbmc/pull/5454 self.JSON_HAS_OPEN_PLAYERCORE_DEFAULT = self.HasJSONCapability("openplayercoredef") # https://github.com/xbmc/xbmc/pull/5324 self.JSON_HAS_LIB_SHOWDIALOGS_PARAM = self.HasJSONCapability("libshowdialogs") #https://github.com/xbmc/xbmc/pull/5786 self.JSON_HAS_EXIT_CODE = self.HasJSONCapability("exitcode") #https://github.com/xbmc/xbmc/pull/7306 self.JSON_HAS_REFRESH_REFACTOR = self.HasJSONCapability("refreshrefactor") # Support profile switching? self.JSON_HAS_PROFILE_SUPPORT = self.HasJSONCapability("profilesupport") #https://github.com/xbmc/xbmc/pull/8080 self.JSON_VOTES_HAVE_NO_GROUPING = self.HasJSONCapability("votesnogrouping") self.JSON_PLAYER_PROCESS_INFO = self.HasJSONCapability("playerprocessinfo") self.JSON_CODEC_INFO_REMOVED = self.HasJSONCapability("codecinforemoved") # https://github.com/xbmc/xbmc/pull/8196 self.JSON_HAS_PROFILE_DIRECTORY = self.HasJSONCapability("profiledirectory") def HasJSONCapability(self, feature): if feature not in self.JSON_VER_CAPABILITIES: raise ValueError("Invalid JSON capability request for feature [%s]" % feature) if self.JSON_VER_CAPABILITIES[feature] == (0, 0, 0): return False else: return self.JSON_VER >= self.JSON_VER_CAPABILITIES[feature] # Call this method once we have worked out if JSON or SQLite will be # used to access textures database def postConfig(self): # Pre-delete artwork if the database filesystem is mounted, as deleting artwork within threads # will result in database locks and inability of the remote client to query its own # database, resulting in invalid responses. if self.getValue(self.config, "download.predelete", "auto") == "auto": if self.USEJSONDB: self.DOWNLOAD_PREDELETE = False else: if self.getDBPath().startswith("/mnt") or self.getDBPath().startswith("\\\\"): ISMOUNT = True else: ISMOUNT = False self.DOWNLOAD_PREDELETE = (os.path.ismount(self.getDBPath()) or ISMOUNT) else: self.DOWNLOAD_PREDELETE = self.getBoolean(self.config, "download.predelete","no") # Depending on JSON version, and user region, lastmodified date will be formatted in several ways. # When JSON_HAS_ISO_DATES, the date will always be YYYY-MM-DD HH:MM:SS, easy. # Otherwise it may be any format... assume either US (mm/dd/yyyy) or non-US (dd/mm/yyyy). self.MDATE_MDY = self.getBoolean(self.config, "modifieddate.mdy", "no") # US (mm/dd/yyyy) or euro (dd/mm/yyyy) if self.JSON_HAS_ISO_DATES: self.qa_lastmodified_fmt = "%Y-%m-%d %H:%M:%S" else: if self.MDATE_MDY: self.qa_lastmodified_fmt = "%m/%d/%Y %H:%M:%S" else: self.qa_lastmodified_fmt = "%d/%m/%Y %H:%M:%S" def getValue(self, config, aKey, default=None, allowundefined=False): value = None try: value = config.get(self.THIS_SECTION, aKey) # If value being undefined is valid, return None for undefined values if not value and allowundefined: return None except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): if self.THIS_SECTION != self.GLOBAL_SECTION: try: value = config.get(self.GLOBAL_SECTION, aKey) # If value being undefined is valid, return None for undefined values if not value and allowundefined: return None except ConfigParser.NoOptionError: if default is None and not allowundefined: raise ConfigParser.NoOptionError(aKey, "%s (or global section)" % self.THIS_SECTION) else: if default is None and not allowundefined: raise ConfigParser.NoOptionError(aKey, self.GLOBAL_SECTION) return value if value else default # default value will be used if key is present, but without a value def getBoolean(self, config, aKey, default="no"): temp = self.getValue(config, aKey, default).lower() return temp in ["yes", "true"] def getSimpleList(self, config, aKey, default="", allowundefined=False, delimiter=",", strip=True): aStr = self.getValue(config, aKey, default, allowundefined) newlist = [] if aStr: for item in [x for x in aStr.split(delimiter) if x]: if item and strip: item = item.strip() if item: newlist.append(item) elif aStr is None and allowundefined: return default return newlist def getFileExtList(self, config, aKey, default="", allowundefined=False): newList = [] for x in [x.lower() for x in self.getSimpleList(config, aKey, default, allowundefined)]: if not x.startswith("."): x = ".%s" % x if x not in newList: newList.append(x) return newList # Return an extended or replacement list def getExRepList(self, config, aKey, initialList=[], allowundefined=False): if self.getValue(config, aKey, "", allowundefined=True) is None: return [] aList = self.getSimpleList(config, aKey, "", allowundefined) iList = initialList if aList: if len(aList) != 0 and aList[0][:1] == "+": aList[0] = aList[0][1:] else: iList = [] for a in aList: if a not in iList: iList.append(a) return iList def getPatternFromList(self, config, aKey, default="", allowundefined=False): aList = self.getValue(config, aKey, default, allowundefined) if aList and aList.startswith("+"): aList = aList[1:] if default and default != "" and aList != "": aList = "%s,%s " % (default, aList.strip()) return [re.compile(x.strip()) for x in aList.split(",") if x] if aList else [] def getListFromPattern(self, aPattern): if not aPattern: return None t = [] for r in aPattern: t.append(r.pattern) return ", ".join(t) def getRelativeDateAndFormat(self, config, key, default, allowundefined=False): adate = self.getValue(config, key, default, allowundefined) if adate: if adate.lower() == "today": t = datetime.date.today() temp_date = datetime.datetime(t.year, t.month, t.day, 0, 0, 0) elif re.search("^[0-9]*$", adate): t = datetime.date.today() - datetime.timedelta(days=int(adate)) temp_date = datetime.datetime(t.year, t.month, t.day, 0, 0, 0) else: temp_date = datetime.datetime.strptime(adate, "%Y-%m-%d %H:%M:%S") date_seconds = MyUtility.SinceEpoch(temp_date) date_formatted = temp_date.strftime("%Y-%m-%d %H:%M:%S") else: date_seconds = None date_formatted = None return (adate, date_seconds, date_formatted) def getQAFields(self, qatype, mediatype, stripModifier=True): if mediatype in ["tvshows", "seasons", "episodes"]: mediatype = "tvshows.%s" % mediatype[:-1] elif mediatype in ["pvr.tv", "pvr.radio", "pvr.channel"]: mediatype = "pvr.%s" % mediatype.split(".")[1] elif mediatype == "tags": mediatype = "movies" key = "qa.%s.%s" % (qatype, mediatype) aStr = self.QA_FIELDS.get(key, None) newlist = [] if aStr: for item in [item.strip() for item in aStr.split(",")]: if item != "": if stripModifier and item[:1] in ["?", "#", "!"]: newitem = item[1:] else: newitem = item if newitem and newitem not in newlist: newlist.append(newitem) return newlist def getFilePath(self, filename=""): if os.path.isabs(self.THUMBNAILS): return os.path.join(self.THUMBNAILS, filename) else: return os.path.join(self.CURRENT_PROFILE["tc.profilepath"], self.THUMBNAILS, filename) def getDBPath(self): if os.path.isabs(self.TEXTUREDB): return self.TEXTUREDB else: return os.path.join(self.CURRENT_PROFILE["tc.profilepath"], self.TEXTUREDB) def NoneIsBlank(self, x): return x if x else "" def BooleanIsYesNo(self, x): return "yes" if x else "no" def dumpJSONCapabilities(self): if self.JSON_VER == (0, 0, 0): return "Unknown, JSON version not acquired" else: caps = {} for feature in self.JSON_VER_CAPABILITIES: caps[feature] = self.HasJSONCapability(feature) return caps # Dump configuration variables, ignoring any keys that are not entirely upper case def dumpMemberVariables(self): mv = {} for key in self.__dict__.keys(): if key == key.upper(): value = self.__dict__[key] if type(value) is list: newlist = [] for v in value: try: newlist.append(v.pattern) except: newlist.append(v) value = newlist mv[key] = value return json.dumps(mv, indent=2, sort_keys=True) def showConfig(self): print("Current properties (if exists, read from %s):" % (self.FILENAME)) print("") print(" sep = %s" % self.FSEP) print(" userdata = %s " % self.KODI_BASE) print(" dbfile = %s" % self.TEXTUREDB) print(" thumbnails = %s " % self.THUMBNAILS) print(" kodi.host = %s" % self.KODI_HOST) print(" webserver.port = %s" % self.WEB_PORT) print(" webserver.ctimeout = %s" % self.NoneIsBlank(self.WEB_CONNECTTIMEOUT)) print(" rpc.port = %s" % self.RPC_PORT) print(" rpc.ipversion = %s" % self.RPC_IPVERSION) print(" rpc.retry = %s" % self.RPC_RETRY) print(" rpc.ctimeout = %s" % self.NoneIsBlank(self.RPC_CONNECTTIMEOUT)) print(" chunked = %s" % self.BooleanIsYesNo(self.CHUNKED)) print(" modifieddate.mdy = %s" % self.BooleanIsYesNo(self.MDATE_MDY)) print(" query.seasons = %s" % self.BooleanIsYesNo(self.QUERY_SEASONS)) print(" query.episodes = %s" % self.BooleanIsYesNo(self.QUERY_EPISODES)) print(" download.predelete = %s" % self.BooleanIsYesNo(self.DOWNLOAD_PREDELETE)) print(" download.payload = %s" % self.BooleanIsYesNo(self.DOWNLOAD_PAYLOAD)) print(" download.retry = %d" % self.DOWNLOAD_RETRY) print(" download.prime = %s" % self.BooleanIsYesNo(self.DOWNLOAD_PRIME)) print(" download.threads = %d" % self.DOWNLOAD_THREADS_DEFAULT) if self.DOWNLOAD_THREADS != {}: for dt in self.DOWNLOAD_THREADS: if self.DOWNLOAD_THREADS[dt] != self.DOWNLOAD_THREADS_DEFAULT: print(" %s = %d" % (dt, self.DOWNLOAD_THREADS[dt])) print(" singlethread.urls = %s" % self.NoneIsBlank(self.getListFromPattern(self.SINGLETHREAD_URLS))) print(" extrajson.addons = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.addons"])) print(" extrajson.agenres = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.agenres"])) print(" extrajson.vgenres = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.vgenres"])) print(" extrajson.albums = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.albums"])) print(" extrajson.artists = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.artists"])) print(" extrajson.songs = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.songs"])) print(" extrajson.movies = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.movies"])) print(" extrajson.sets = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.sets"])) print(" extrajson.tvshows.tvshow = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.tvshows.tvshow"])) print(" extrajson.tvshows.season = %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.tvshows.season"])) print(" extrajson.tvshows.episode= %s" % self.NoneIsBlank(self.XTRAJSON["extrajson.tvshows.episode"])) print(" setmembers = %s" % self.BooleanIsYesNo(self.ADD_SET_MEMBERS)) print(" songmembers = %s" % self.BooleanIsYesNo(self.ADD_SONG_MEMBERS)) print(" qaperiod = %d (added after %s)" % (self.QAPERIOD, self.QADATE)) print(" qa.file = %s" % self.BooleanIsYesNo(self.QA_FILE)) print(" qa.nfo.refresh = %s%s" % (self.NoneIsBlank(self.QA_NFO_REFRESH), " (%s)" % self.qa_nfo_refresh_date_fmt if self.qa_nfo_refresh_date_fmt else "")) print(" qa.useoldrefreshmethod = %s" % (self.BooleanIsYesNo(self.QA_USEOLDREFRESHMETHOD))) print(" qa.fail.checkexists = %s" % self.BooleanIsYesNo(self.QA_FAIL_CHECKEXISTS)) print(" qa.fail.missinglocalart = %s" % self.BooleanIsYesNo(self.QA_FAIL_MISSING_LOCAL_ART)) print(" qa.fail.urls = %s" % self.NoneIsBlank(self.getListFromPattern(self.QA_FAIL_TYPES))) print(" qa.warn.urls = %s" % self.NoneIsBlank(self.getListFromPattern(self.QA_WARN_TYPES))) for k in sorted(self.QA_FIELDS): print(" %s = %s" % (k, self.NoneIsBlank(self.QA_FIELDS[k]))) print(" cache.castthumb = %s" % self.BooleanIsYesNo(self.CACHE_CAST_THUMB)) print(" cache.hideallitems = %s" % self.BooleanIsYesNo(self.CACHE_HIDEALLITEMS)) print(" cache.artwork = %s" % self.NoneIsBlank(", ".join(self.CACHE_ARTWORK))) print(" cache.ignore.types = %s" % self.NoneIsBlank(self.getListFromPattern(self.CACHE_IGNORE_TYPES))) print(" cache.extrafanart = %s" % self.BooleanIsYesNo(self.CACHE_EXTRA_FANART)) print(" cache.extrathumbs = %s" % self.BooleanIsYesNo(self.CACHE_EXTRA_THUMBS)) print(" cache.videoextras = %s" % self.BooleanIsYesNo(self.CACHE_VIDEO_EXTRAS)) print(" cache.refresh = %s%s" % (self.NoneIsBlank(self.CACHE_REFRESH), " (%s)" % self.cache_refresh_date_fmt if self.cache_refresh_date_fmt else "")) print(" cache.dropfile = %s" % self.NoneIsBlank(self.CACHE_DROP_INVALID_FILE)) print(" prune.retain.types = %s" % self.NoneIsBlank(self.getListFromPattern(self.PRUNE_RETAIN_TYPES))) print(" prune.retain.previews = %s" % self.BooleanIsYesNo(self.PRUNE_RETAIN_PREVIEWS)) print(" prune.retain.pictures = %s" % self.BooleanIsYesNo(self.PRUNE_RETAIN_PICTURES)) print(" prune.retain.chapters = %s" % self.BooleanIsYesNo(self.PRUNE_RETAIN_CHAPTERS)) print(" missing.ignore.patterns = %s" % self.NoneIsBlank(self.getListFromPattern(self.MISSING_IGNORE_PATTERNS))) print(" logfile = %s" % self.NoneIsBlank(self.LOGFILE)) print(" logfile.verbose = %s" % self.BooleanIsYesNo(self.LOGVERBOSE)) print(" logfile.unique = %s" % self.BooleanIsYesNo(self.LOGUNIQUE)) print(" logfile.dcache = %s" % self.BooleanIsYesNo(self.LOGDCACHE)) print(" checkupdate = %s" % self.BooleanIsYesNo(self.CHECKUPDATE)) print(" autoupdate = %s" % self.BooleanIsYesNo(self.AUTOUPDATE)) if self.RECACHEALL: print(" allow.recacheall = yes") temp = " (%s)" % self.LASTRUNFILE_DATETIME if self.LASTRUNFILE and self.LASTRUNFILE_DATETIME else "" print(" lastrunfile = %s%s" % (self.NoneIsBlank(self.LASTRUNFILE), temp)) print(" orphan.limit.check = %s" % self.BooleanIsYesNo(self.ORPHAN_LIMIT_CHECK)) print(" purge.minlen = %s" % self.PURGE_MIN_LEN) print(" picture.filetypes = %s" % self.NoneIsBlank(", ".join(self.PICTURE_FILETYPES_EX))) print(" video.filetypes = %s" % self.NoneIsBlank(", ".join(self.VIDEO_FILETYPES_EX))) print(" audio.filetypes = %s" % self.NoneIsBlank(", ".join(self.AUDIO_FILETYPES_EX))) print(" subtitle.filetypes = %s" % self.NoneIsBlank(", ".join(self.SUBTITLE_FILETYPES_EX))) print(" watched.overwrite = %s" % self.BooleanIsYesNo(self.WATCHEDOVERWRITE)) print(" network.mac = %s" % self.NoneIsBlank(self.MAC_ADDRESS)) print(" imdb.fields.movies = %s" % self.NoneIsBlank(", ".join(self.IMDB_FIELDS_MOVIES))) print(" imdb.fields.tvshows = %s" % self.NoneIsBlank(", ".join(self.IMDB_FIELDS_TVSHOWS))) print(" imdb.ignore.tvtitles = %s" % self.NoneIsBlank("|".join(self.IMDB_IGNORE_TVTITLES))) print(" imdb.map.tvtitles = %s" % self.NoneIsBlank("|".join(self.IMDB_MAP_TVTITLES))) print(" imdb.translate.tvtitles = %s" % self.NoneIsBlank("|".join(self.IMDB_TRANSLATE_TVTITLES))) print(" imdb.translate.tvyears = %s" % self.NoneIsBlank("|".join(self.IMDB_TRANSLATE_TVYEARS))) print(" imdb.ignore.missing.episodes = %s" % self.BooleanIsYesNo(self.IMDB_IGNORE_MISSING_EPISODES)) print(" imdb.threads = %s" % self.IMDB_THREADS) print(" imdb.timeout = %s" % self.NoneIsBlank(self.IMDB_TIMEOUT)) print(" imdb.retry = %s" % self.IMDB_RETRY) print(" imdb.grouping = %s" % self.IMDB_GROUPING) if self.IMDB_PERIOD_FROM: print(" imdb.period = %s (added after %s)" % (self.NoneIsBlank(self.IMDB_PERIOD), self.NoneIsBlank(self.IMDB_PERIOD_FROM))) else: print(" imdb.period = %s" % (self.NoneIsBlank(self.IMDB_PERIOD))) print(" imdb.delete_parenthesis = %s" % self.BooleanIsYesNo(self.IMDB_DEL_PARENTHESIS)) print(" bin.tvservice = %s" % self.NoneIsBlank(self.BIN_TVSERVICE)) print(" bin.vcgencmd = %s" % self.NoneIsBlank(self.BIN_VCGENCMD)) print(" bin.ceccontrol = %s" % self.NoneIsBlank(self.BIN_CECCONTROL)) print(" hdmi.force.hotplug = %s" % self.BooleanIsYesNo(self.HDMI_FORCE_HOTPLUG)) print(" hdmi.ignoresuspend = %s" % self.BooleanIsYesNo(self.HDMI_IGNORE_SUSPEND)) print(" hdmi.ignoredisable = %s" % self.BooleanIsYesNo(self.HDMI_IGNORE_DISABLE)) print(" hdmi.ignoreplayer = %s" % self.BooleanIsYesNo(self.HDMI_IGNORE_PLAYER)) print(" hdmi.ignorelibrary = %s" % self.BooleanIsYesNo(self.HDMI_IGNORE_LIBRARY)) print(" dcache.size = %d" % self.DCACHE_SIZE) print(" dcache.agelimit = %d" % self.DCACHE_AGELIMIT) print(" posterwidth = %d" % self.POSTER_WIDTH) print(" clean.showdialogs = %s" % self.BooleanIsYesNo(self.CLEAN_SHOW_DIALOGS)) print(" scan.showdialogs = %s" % self.BooleanIsYesNo(self.SCAN_SHOW_DIALOGS)) # print(" profile.master = %s" % self.NoneIsBlank(self.PROFILE_MASTER)) print(" profile.enabled = %s" % self.BooleanIsYesNo(self.PROFILE_ENABLED)) print(" profile.autoload = %s" % self.BooleanIsYesNo(self.PROFILE_AUTOLOAD)) print(" profile.retry = %s" % self.PROFILE_RETRY) print(" profile.wait = %s" % self.PROFILE_WAIT) print(" profile.name = %s" % self.NoneIsBlank(self.PROFILE_NAME)) print(" profile.password = %s" % self.NoneIsBlank(self.PROFILE_PASSWORD)) print(" profile.password.encrypted = %s" % self.BooleanIsYesNo(self.PROFILE_ENCRYPTED)) print(" profile.directory = %s" % self.NoneIsBlank(self.PROFILE_DIRECTORY)) print("") print("See http://wiki.xbmc.org/index.php?title=JSON-RPC_API/v6 for details of available audio/video fields.") # # Very simple logging class. Will be a global object, also passed to threads # hence the Lock() methods.. # # Writes progress information to stderr so that # information can still be grep'ed easily (stdout). # # Prefix logfilename with + to enable flushing after each write. # class MyLogger(): def __init__(self): self.lastlen = 0 self.now = 0 self.LOGGING = False self.LOGFILE = None self.LOGFLUSH = False self.DEBUG = False self.VERBOSE = False try: self.ISATTY = sys.stdout.isatty() except: self.ISATTY = False #Ensure stdout/stderr use utf-8 encoding... if MyUtility.isPython3_1: sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) sys.stderr = codecs.getwriter("utf-8")(sys.stderr.detach()) else: sys.stdout = codecs.getwriter("utf-8")(sys.stdout) sys.stderr = codecs.getwriter("utf-8")(sys.stderr) def __del__(self): if self.LOGFILE: self.LOGFILE.close() def setLogFile(self, config=None): with lock: if config and config.LOGFILE: if not self.LOGGING: filename = config.LOGFILE self.LOGFLUSH = filename.startswith("+") if self.LOGFLUSH: filename = filename[1:] try: if config.LOGUNIQUE: t = tempfile.mkstemp(prefix="%s." % os.path.basename(filename), suffix="", dir=os.path.dirname(filename)) filename = t[1] os.close(t[0]) self.LOGFILE = codecs.open(filename, "w", encoding="utf-8") self.LOGGING = True except: raise IOError("Unable to open logfile for writing!") else: self.LOGGING = False if self.LOGFILE: self.LOGFILE.close() self.LOGFILE = None def progress(self, data, every=0, finalItem=False, newLine=False, noBlank=False): with lock: if every != 0 and not finalItem: self.now += 1 if self.now != 1: if self.now <= every: return else: self.reset(initialValue=1) else: self.reset(initialValue=0) udata = MyUtility.toUnicode(data) ulen = len(data) spaces = self.lastlen - ulen self.lastlen = ulen if spaces > 0 and not noBlank: sys.stderr.write("%-s%*s\r%s" % (udata, spaces, " ", ("\n" if newLine else ""))) else: sys.stderr.write("%-s\r%s" % (udata, ("\n" if newLine else ""))) if newLine: self.lastlen = 0 sys.stderr.flush() def reset(self, initialValue=0): self.now = initialValue def out(self, data, newLine=False, log=False, padspaces=True): with lock: udata = MyUtility.toUnicode(data) ulen = len(data) spaces = self.lastlen - ulen self.lastlen = ulen if udata.rfind("\n") == -1 else 0 NL = "\n" if newLine else "" if spaces > 0: sys.stdout.write("%-s%*s%s" % (udata, spaces, " ", NL)) else: sys.stdout.write("%-s%s" % (udata, NL)) if newLine or not padspaces: self.lastlen = 0 try: sys.stdout.flush() except IOError as e: if e.errno == errno.EPIPE: pass else: raise if log: self.log(data) def debug(self, data, jsonrequest=None): with lock: if self.DEBUG: if self.ISATTY: self.out("%s: [%s] %s" % (datetime.datetime.now(), self.OPTION, data), newLine=True) else: self.out("[%s] %s" % (self.OPTION, data), newLine=True) if self.LOGGING: self.log("[DEBUG] %s" % data, jsonrequest=jsonrequest) def log(self, data, jsonrequest=None, maxLen=0): if self.LOGGING: with lock: udata = MyUtility.toUnicode(data) t = threading.current_thread().name if jsonrequest is None: d = udata if maxLen != 0 and not self.VERBOSE and len(d) > maxLen: d = "%s (truncated)" % d[:maxLen] self.LOGFILE.write("%s:%-10s: %s\n" % (datetime.datetime.now(), t, d)) else: d = json.dumps(jsonrequest, ensure_ascii=True) if maxLen != 0 and len(d) > maxLen: d = "%s (truncated)" % d[:maxLen] self.LOGFILE.write("%s:%-10s: %s [%s]\n" % (datetime.datetime.now(), t, udata, d)) if self.DEBUG or self.LOGFLUSH: self.LOGFILE.flush() # Use this method for large Unicode data - tries to minimize # creation of additional temporary buffers through concatenation. def log2(self, prefix, udata, jsonrequest=None, maxLen=0): if self.LOGGING: with lock: t = threading.current_thread().name self.LOGFILE.write("%s:%-10s: %s" % (datetime.datetime.now(), t, prefix)) if jsonrequest is None: if maxLen != 0 and not self.VERBOSE and len(udata) > maxLen: d = "%s (truncated)" % udata[:maxLen] self.LOGFILE.write(d) else: self.LOGFILE.write(udata) else: d = json.dumps(jsonrequest, ensure_ascii=True) if maxLen != 0 and len(d) > maxLen: d = "%s (truncated)" % d[:maxLen] self.LOGFILE.write("[") self.LOGFILE.write(d) self.LOGFILE.write("]") self.LOGFILE.write("\n") if self.DEBUG or self.LOGFLUSH: self.LOGFILE.flush() def err(self, data, newLine=False, log=False): with lock: self.progress("") udata = MyUtility.toUnicode(data) if newLine: sys.stderr.write("%-s\n" % udata) else: sys.stderr.write("%-s" % udata) sys.stderr.flush() if log: self.log(data) def flush(self): sys.stdout.flush() sys.stderr.flush() if self.LOGFILE: self.LOGFILE.flush() # # Image loader thread class. # class MyImageLoader(threading.Thread): def __init__(self, work_queue, other_queue, error_queue, complete_queue, config, logger, totals, force=False, retry=0): threading.Thread.__init__(self) self.work_queue = work_queue self.other_queue = other_queue self.error_queue = error_queue self.complete_queue = complete_queue self.config = config self.logger = logger self.database = MyDB(config, logger) self.json = MyJSONComms(config, logger) self.totals = totals self.force = force self.retry = retry self.totals.init(self.name) def run(self): with self.database: while not stopped.is_set(): try: item = self.work_queue.get(block=False) self.work_queue.task_done() if not self.loadImage(item) and not item.missingOK: self.error_queue.put(item) self.complete_queue.put(item) except Queue.Empty: break except IOEndOfReplayLog: break self.totals.stop() self.complete_queue.put(None) def geturl(self, item): PDRETRY = self.retry # Call Files.PrepareDownload. If failure, retry up to retry times, waiting a short # interval between each attempt. url = self.json.getDownloadURL(item.filename) rowexists = True # If no URL, could be because thumbnail is missing but DB row exists - if thumbnail # no longer available, then delete the row and try again to obtain URL if url is None and not self.config.DOWNLOAD_PREDELETE and item.dbid != 0 and self.force: if self.config.HAS_THUMBNAILS_FS and not os.path.exists(self.config.getFilePath(item.cachedurl)): self.logger.log("Deleting row with missing image from cache - id [%d], cachedurl [%s] for filename [%s]" % (item.dbid, item.cachedurl, item.decoded_filename)) self.database.deleteItem(item.dbid, None) rowexists = False # If DOWNLOAD_PRIME is enabled, request the remote URL directly. If not available, don't bother # retrying call to Files.PrepareDownload as it will surely fail. if PDRETRY > 0 and url is None and self.config.DOWNLOAD_PRIME: isAvailable = self.prime_the_request(item.decoded_filename) else: isAvailable = True # Retry call to Files.PrepareDownload - hopefully it will succeed eventually... while PDRETRY > 0 and url is None and isAvailable: self.logger.log("Retrying getDownloadURL(), %d attempts remaining" % PDRETRY) # Introduce a short delay, unless we're playing back a log file... if self.json.LOG_REPLAYFILE is None: time.sleep(0.5) PDRETRY -= 1 url = self.json.getDownloadURL(item.filename) return (url, rowexists) # Directly request the remote URL returning True if still available def prime_the_request(self, url): if url is None: return False if url.startswith("http://"): domain = url.replace("http://", "").split("/")[0] page = "/" + "/".join(url.replace("http://", "").split("/")[1:]) issecure = False elif url.startswith("https://"): domain = url.replace("https://", "").split("/")[0] page = "/" + "/".join(url.replace("https://", "").split("/")[1:]) issecure = True else: return True isAvailable = True try: PAYLOAD = self.json.sendWeb("GET", page, "primeImage", headers={"User-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0"}, \ readAmount=1024, rawData=True, domain=domain, useSSL=issecure) self.logger.log("Primed request of: HTTPS=%s, Domain [%s] with URL [%s], result [%d, %s]" % (issecure, domain, page, self.json.WEB_LAST_STATUS, self.json.WEB_LAST_REASON)) isAvailable = (self.json.WEB_LAST_STATUS == 200 or 300 <= self.json.WEB_LAST_STATUS < 400) except: isAvailable = False return isAvailable def loadImage(self, item): ATTEMPT = 1 if self.retry < 1 else self.retry PERFORM_DOWNLOAD = False self.totals.start(item.mtype, item.itype) (url, rowexists) = self.geturl(item) if url: if not self.config.DOWNLOAD_PREDELETE: if item.dbid != 0 and self.force: if rowexists: self.logger.log("Deleting old image from cache with id [%d], cachedurl [%s] for filename [%s]" % (item.dbid, item.cachedurl, item.decoded_filename)) self.database.deleteItem(item.dbid, item.cachedurl) self.totals.bump("Deleted", item.itype) PERFORM_DOWNLOAD = True if self.config.DOWNLOAD_PAYLOAD or PERFORM_DOWNLOAD: self.logger.log("Proceeding with download of URL [%s]" % url) else: self.logger.log("Image not available for download - uncacheable (embedded?), or doesn't exist. Filename [%s]" % item.filename) ATTEMPT = 0 while ATTEMPT > 0 and (self.config.DOWNLOAD_PAYLOAD or PERFORM_DOWNLOAD): try: # Don't need to download the whole image for it to be cached so just grab the first 1KB PAYLOAD = self.json.sendWeb("GET", url, "loadImage", readAmount=1024, rawData=True) if self.json.WEB_LAST_STATUS == httplib.OK: self.logger.log("Successfully downloaded image with size [%d] bytes, attempts required [%d], filename [%s]" \ % (len(PAYLOAD), (self.retry - ATTEMPT + 1), item.decoded_filename)) break except: pass ATTEMPT -= 1 self.logger.log("Failed to download image URL [%s], status [%d], " \ "attempts remaining [%d]" % (url, self.json.WEB_LAST_STATUS, ATTEMPT)) if stopped.is_set(): ATTEMPT = 0 if ATTEMPT == 0: if not item.missingOK: self.totals.bump("Error", item.itype) else: self.totals.bump("Cached", item.itype) self.totals.finish(item.mtype, item.itype) return ATTEMPT != 0 # # IMDB Thread # class MyIMDBLoader(threading.Thread): def __init__(self, config, logger, input_queue, output_queue, plotFull, plotOutline, movies250, imdbfields): threading.Thread.__init__(self) self.config = config self.logger = logger self.input_queue = input_queue self.output_queue = output_queue self.plotFull = plotFull self.plotOutline = plotOutline self.movies250 = movies250 # We don't need to query OMDb if only updating top250 self.omdbquery = (imdbfields != ["top250"]) # Avoid querying OMDb if we're only updating fields available from Top250 movie list self.onlyt250fields = (set(imdbfields).issubset(["top250", "rating", "votes"])) self.ignore_missing_episodes = config.IMDB_IGNORE_MISSING_EPISODES self.timeout = self.config.IMDB_TIMEOUT self.retry = self.config.IMDB_RETRY self.apikey = self.config.OMDB_API_KEY def run(self): while not stopped.is_set(): try: qItem = self.input_queue.get(block=False) self.input_queue.task_done() item = qItem["item"] imdbnumber = item.get("imdbnumber", None) title = item.get("title", item["label"]) original_title = qItem.get("OriginalShowTitle", title) year = item.get("year", None) multipart_ep = qItem.get("multipart_ep", None) # (season#, episode#) of first episode in a multipart episode showTitle = title showYear = year season = None episode = None isMovie = isTVShow = isEpisode = False if "movieid" in item: libid = item["movieid"] isMovie = True elif "tvshowid" in item: libid = item["tvshowid"] if year and title.endswith("(%d)" % year): title = re.sub("\(%d\)$" % year, "", title).strip() isTVShow = True elif "episodeid" in item: libid = item["episodeid"] show_title = qItem["ShowTitle"] show_year = qItem["ShowYear"] season = qItem["Season"] episode = qItem["Episode"] isEpisode = True if isMovie and self.movies250 is not None and imdbnumber is not None: movie250 = self.movies250.get(imdbnumber, {}) # No need to query OMDb if all Top250 fields are available and they're all we need needomdb = not (movie250 is not {} and self.onlyt250fields and ("rank" in movie250 and "rating" in movie250 and "votes" in movie250)) else: movie250 = {} needomdb = True if self.omdbquery and needomdb: attempt = 0 newimdb = None ismultipartquery = False while True: if isMovie: self.logger.log("Querying OMDb [a=%d, r=%d]: [movie] %s, imdb=%s" % (attempt, self.retry, imdbnumber, title)) newimdb = MyUtility.getIMDBInfo("movie", self.apikey, imdbnumber=imdbnumber, plotFull=self.plotFull, plotOutline=self.plotOutline, qtimeout=self.timeout) if imdbnumber else None elif isTVShow: self.logger.log("Querying OMDb [a=%d, r=%d]: [tvshow] %s, %d, imdb=%s" % (attempt, self.retry, title, year, MyUtility.nonestr(imdbnumber))) newimdb = MyUtility.getIMDBInfo("tvshow", self.apikey, imdbnumber=imdbnumber, title=title, year=year, plotFull=self.plotFull, plotOutline=self.plotOutline, qtimeout=self.timeout) elif isEpisode: self.logger.log("Querying OMDb [a=%d, r=%d]: [episode] %s, %d, S%02d, E%02d, imdb=%s" % (attempt, self.retry, show_title, show_year, season, episode, MyUtility.nonestr(imdbnumber))) newimdb = MyUtility.getIMDBInfo("episode", self.apikey, imdbnumber=imdbnumber, title=show_title, year=show_year, season=season, episode=episode, plotFull=(self.plotFull and not ismultipartquery), plotOutline=(self.plotOutline and not ismultipartquery), qtimeout=self.timeout) if newimdb is not None: newimdb = newimdb if newimdb.get("response", "False") == "True" else None # If nothing found and it's a multipart episode that hasn't already been re-queried, then # try querying omdbapi again using the first episode of the multipart sequence (ignoring plot details) if newimdb is None and multipart_ep is not None and not ismultipartquery: ismultipartquery = True self.logger.log("Re-querying OMDb [a=%d, r=%d]: [multipart] %s, %d, S%02d, E%02d => S%02d, E%02d" % (attempt, self.retry, show_title, show_year, season, episode, multipart_ep[0], multipart_ep[1])) (season, episode) = multipart_ep continue break elif attempt >= self.retry: break attempt += 1 if newimdb is None: if isMovie: logmsg = "Could not obtain OMDb details for [movie] %s (%s)" % (imdbnumber, original_title) elif isTVShow: logmsg = "Could not obtain OMDb details for [tvshow] %s (%d)" % (original_title, year) elif isEpisode: logmsg = "Could not obtain OMDb details for [episode] %s S%02dE%02d" % (original_title, season, episode) if isEpisode and self.ignore_missing_episodes: self.logger.log(logmsg) else: self.logger.err(logmsg, newLine=True, log=True) continue else: self.logger.log("Avoided OMDb query, movies250 has all we need: %s (%s)" % (imdbnumber, original_title)) newimdb = {} # Add top250 only if we've got a 250 list if self.movies250 is not None: newimdb["top250"] = movie250.get("rank", 0) newimdb["rating"] = movie250.get("rating", newimdb.get("rating",None)) newimdb["votes"] = movie250.get("votes", newimdb.get("votes", None)) qItem["newimdb"] = newimdb self.output_queue.put(qItem) except Queue.Empty: self.output_queue.put(None) break # # Simple thread class to manage Raspberry Pi HDMI power state # class MyHDMIManager(threading.Thread): def __init__(self, config, logger, cmdqueue, hdmidelay=900, onstopdelay=5): threading.Thread.__init__(self) self.EV_PLAY_STOP = "play.stop" self.EV_HDMI_OFF = "hdmi.off" self.events = {} self.config = config self.logger = logger self.cmdqueue = cmdqueue self.ignoreplayer = self.config.HDMI_IGNORE_PLAYER self.ignorelibrary = self.config.HDMI_IGNORE_LIBRARY self.bin_tvservice = config.BIN_TVSERVICE self.bin_vcgencmd = config.BIN_VCGENCMD if config.BIN_VCGENCMD and os.path.exists(config.BIN_VCGENCMD) else None self.bin_ceccontrol = config.BIN_CECCONTROL if config.BIN_CECCONTROL and os.path.exists(config.BIN_CECCONTROL) else None hdmidelay = 0 if hdmidelay < 0 else hdmidelay onstopdelay = 5 if onstopdelay < 5 else onstopdelay # Order of event processing is important, as # we want to process the hdmi.off event after # all other events. # # Higher numbers for order are processed last. # self.EventAdd(name=self.EV_PLAY_STOP, delayTime=onstopdelay, order=1) self.EventAdd(name=self.EV_HDMI_OFF, delayTime=hdmidelay, order=99) self.logger.debug("HDMI Power off delay: %d seconds (ignored when CanSuspend is yes)" % self.EventInterval(self.EV_HDMI_OFF)) self.logger.debug("Player OnStop delay : %d seconds (ignored when CanSuspend is yes)" % self.EventInterval(self.EV_PLAY_STOP)) self.logger.debug("Path to tvservice : %s" % self.bin_tvservice) self.logger.debug("Path to vcgencmd : %s" % self.bin_vcgencmd) self.logger.debug("Path to ceccontrol : %s" % self.bin_ceccontrol) self.logger.debug("Ignore Active Player: %s" % ("Yes" if self.ignoreplayer else "No")) self.logger.debug("Ignore Library Scan : %s" % ("Yes" if self.ignorelibrary else "No")) def run(self): try: self.MonitorKodi() except: pass def MonitorKodi(self): clientState = {} hdmi_on = True screensaver_active = False player_active = False library_active = False self.cansuspend = False self.candisable = False qtimeout = None # This is the order events will be processed... ordered_event_keys = [x[0] for x in sorted(self.events.items(), key=lambda e: e[1]["event.order"])] while not stopped.is_set(): try: notification = self.cmdqueue.get(block=True, timeout=qtimeout) self.cmdqueue.task_done() method = notification["method"] params = notification["params"] if method == "pong": self.logger.debug("Connected to Kodi") self.logger.debug("HDMI power management thread - initialising Kodi and HDMI state") clientState = self.getKodiStatus() hdmi_on = self.getHDMIState() screensaver_active = clientState["screensaver.active"] player_active = clientState["players.active"] library_active = (clientState["scanning.music"] or clientState["scanning.video"]) # If the Pi can self-suspend, don't schedule the EV_HDMI_OFF event or restart Kodi # to re-init the HDMI. Instead, just log various events and call ceccontrol # whenever sleeping or waking. self.cansuspend = clientState["cansuspend"] # If firmware supports ability to disable HDMI power, then use "vcgencmd display_power 0|1" # to control HDMI rather than tvservice. self.candisable = clientState["vcgencmd.display_power"] if hdmi_on == False: hdmi_status = "off" elif self.candisable and not self.getPowerState(): hdmi_status = "disabled" hdmi_on = False else: hdmi_status = "on" self.logger.debug("HDMI is [%s], Screensaver is [%s], Player is [%s], Library scan [%s], CanSuspend [%s], CanDisable [%s]" % (hdmi_status, ("active" if screensaver_active else "inactive"), ("active" if player_active else "inactive"), ("active" if library_active else "inactive"), ("yes" if self.cansuspend else "no"), ("yes" if self.candisable else "no") )) if not self.cansuspend: if screensaver_active and hdmi_on: self.EventSet(self.EV_HDMI_OFF) elif method == "GUI.OnScreensaverActivated": self.logger.debug("Screensaver has activated") screensaver_active = True if not self.cansuspend: self.EventSet(self.EV_HDMI_OFF) elif method == "GUI.OnScreensaverDeactivated": self.logger.debug("Screensaver has deactivated") screensaver_active = False if not self.cansuspend: if self.EventEnabled(self.EV_HDMI_OFF): self.EventStop(self.EV_HDMI_OFF) self.logger.debug("Scheduled HDMI power-off cancelled") else: hdmi_on = self.enable_hdmi() if not self.candisable: self.sendKodiExit() elif method == "Player.OnStop": self.EventSet(self.EV_PLAY_STOP) elif method == "Player.OnPlay": if not player_active: self.logger.debug("Player has started") player_active = True self.EventStop(self.EV_PLAY_STOP) elif method.endswith(".OnScanStarted") or method.endswith(".OnCleanStarted"): self.logger.debug("Library scan has started") library_active = True elif method.endswith(".OnScanFinished") or method.endswith(".OnCleanFinished"): self.logger.debug("Library scan has finished") library_active = False elif method == "System.OnSleep" and self.cansuspend: self.logger.debug("Client is now suspended") # HDMI already disabled, but may need to process CEC functionality self.callCECControl("off") elif method == "System.OnWake" and self.cansuspend: self.logger.debug("Client has resumed") # HDMI already enabled, but may need to process CEC functionality self.callCECControl("on") elif method == "System.OnQuit": self.EventsStopAll() if not hdmi_on: hdmi_on = self.enable_hdmi() except Queue.Empty as e: pass qtimeout = None # Process events once queue is empty of all notifications if self.cmdqueue.empty(): now = time.time() for event in ordered_event_keys: # Start any pending events if self.EventPending(event): self.EventStart(event, now) if event == self.EV_HDMI_OFF: self.logger.debug("HDMI power off in %d seconds unless cancelled" % int(self.EventInterval(event))) if (player_active and not self.ignoreplayer) or (library_active and not self.ignorelibrary): self.logger.debug("HDMI power-off will not occur until both player and library become inactive") # Process any expired events if self.EventExpired(event, now): if event == self.EV_PLAY_STOP: self.logger.debug("Player has stopped") player_active = False self.EventStop(event) elif event == self.EV_HDMI_OFF: if (player_active and not self.ignoreplayer) or (library_active and not self.ignorelibrary): if not self.EventOverdue(event, now): self.logger.debug("HDMI power-off timeout reached - waiting for player and/or library to become inactive") else: hdmi_on = self.disable_hdmi() self.EventStop(self.EV_HDMI_OFF) # Block until the next scheduled event, or block # indefinitely if no events are currently scheduled if self.EventEnabled(event) and not self.EventOverdue(event, now): nextevent = self.EventInterval(event, now) qtimeout = nextevent if nextevent >= 0 and (not qtimeout or nextevent < qtimeout) else qtimeout def EventExpired(self, name, currentTime): if self.events[name]["start"] != 0: return (self.EventInterval(name, currentTime) <= 0) else: return False def EventInterval(self, name, currentTime=None): if not currentTime or self.events[name]["start"] == 0: return self.events[name]["timeout"] else: t = self.events[name]["start"] + self.events[name]["timeout"] if currentTime: return t - currentTime else: return t - time.time() def EventEnabled(self, name): return self.events[name]["start"] != 0 def EventPending(self, name): return self.events[name]["pending"] def EventOverdue(self, name, currentTime=None): overdue = self.events[name]["overdue"] if not overdue: if self.EventExpired(name, currentTime): self.events[name]["overdue"] = True return overdue def EventAdd(self, name, delayTime, order=None): self.events[name] = {"event.order": order, "timeout": delayTime} self.EventStop(name) def EventSet(self, name): self.events[name]["pending"] = True def EventStart(self, name, currentTime): self.events[name]["start"] = currentTime self.events[name]["pending"] = False def EventStop(self, name): self.events[name]["start"] = 0 self.events[name]["overdue"] = False self.events[name]["pending"] = False def EventsStopAll(self): for event in self.events: self.EventStop(event) def sendKodiExit(self): self.logger.debug("Sending Application.Quit() to Kodi") REQUEST = {"method": "Application.Quit"} MyJSONComms(self.config, self.logger).sendJSON(REQUEST, "libExit", checkResult=False) def getKodiStatus(self): jcomms = MyJSONComms(self.config, self.logger) statuses = {} REQUEST = {"method": "XBMC.GetInfoBooleans", "params": {"booleans": ["System.ScreenSaverActive", "Library.IsScanningMusic", "Library.IsScanningVideo"]}} data = jcomms.sendJSON(REQUEST, "libBooleans", checkResult=False) values = data.get("result", {}) statuses["screensaver.active"] = values.get("System.ScreenSaverActive", False) statuses["scanning.music"] = values.get("Library.IsScanningMusic", False) statuses["scanning.video"] = values.get("Library.IsScanningVideo", False) REQUEST = {"method": "Player.GetActivePlayers"} data = jcomms.sendJSON(REQUEST, "libPlayers", checkResult=False) values = data.get("result", []) statuses["players.active"] = (values != []) REQUEST = {"method": "System.GetProperties", "params": {"properties": ["canshutdown", "cansuspend", "canreboot", "canhibernate"]}} data = jcomms.sendJSON(REQUEST, "libProperties", checkResult=False) values = data.get("result", {}) for s in REQUEST["params"]["properties"]: statuses[s] = values.get(s, False) statuses["cansuspend"] = (statuses["cansuspend"] and not self.config.HDMI_IGNORE_SUSPEND) svalue = False if self.bin_vcgencmd: try: response = subprocess.check_output([self.bin_vcgencmd, "commands"], stderr=subprocess.STDOUT).decode("utf-8") response = response[:-1] if response.endswith("\n") else response if response.find("display_power") != -1: svalue = True except: pass statuses["vcgencmd.display_power"] = (svalue and not self.config.HDMI_IGNORE_DISABLE) return statuses # Determine if TV is on or off - if off, no need to disable HDMI # True = ON, False = OFF # # The hotplug shows in state: # VC_HDMI_UNPLUGGED = (1 << 0), /**= 13: SQL = "SELECT t.id, t.cachedurl, t.lasthashcheck, t.url, s.height, s.width, s.usecount, s.lastusetime, s.size, t.imagehash " \ "FROM texture t JOIN sizes s ON (t.id = s.idtexture)" else: SQL = "SELECT t.id, t.cachedurl, t.lasthashcheck, t.url, 0 as height, 0 as width, t.usecount, t.lastusetime, 0 as size, t.imagehash " \ "FROM texture t" if filter: SQL = "%s %s" % (SQL, filter) if order: SQL = "%s %s" % (SQL, order) return self.execute(SQL).fetchall() # Return SQLite database rows as a dictionary list # to match JSON equivalent def _transform(self, rows): data = [] funcNormalise = MyUtility.normalise if rows: for r in rows: url = funcNormalise(r[3], strip=True) data.append({u"textureid": r[0], u"cachedurl": r[1], u"lasthashcheck": r[2], u"url": url, u"sizes":[{u"height": r[4], u"width": r[5], u"usecount": r[6], u"lastused": r[7], u"size": r[8]}], u"imagehash": r[9]}) return data def delRowByID(self, id): if id > 0: if self.usejson: data = self.mydb.delTexture(id) if not ("result" in data and data["result"] == "OK"): self.logger.out("id %s is not valid\n" % (self.config.IDFORMAT % int(id))) else: self.execute("DELETE FROM texture WHERE id=%d" % id) self.getDB().commit() def deleteItem(self, id, cachedURL=None, warnmissing=True): # When deleting rows via JSON, the artwork file and also # any corresponding DDS file should also be removed if self.usejson and id > 0: self.delRowByID(id) return if cachedURL is not None and id > 0: row = self.getSingleRow("WHERE id = %d" % id) if row is None: self.logger.out("id %s is not valid\n" % (self.config.IDFORMAT % int(id))) return else: localFile = row["cachedurl"] else: localFile = cachedURL if localFile is not None and os.path.exists(self.config.getFilePath(localFile)): os.remove(self.config.getFilePath(localFile)) self.logger.log("FILE DELETE: Removed cached thumbnail file %s for id %s" % (localFile, (self.config.IDFORMAT % id))) else: if warnmissing: self.logger.out("WARNING: id %s, cached thumbnail file %s not found" % ((self.config.IDFORMAT % id), localFile), newLine=True) # Check for any matching .dds file and remove that too if localFile: localFile_dds = "%s.dds" % os.path.splitext(localFile)[0] if localFile_dds and os.path.exists(self.config.getFilePath(localFile_dds)): os.remove(self.config.getFilePath(localFile_dds)) self.logger.log("FILE DELETE: Removed cached thumbnail file %s for id %s" % (localFile_dds, (self.config.IDFORMAT % id))) self.delRowByID(id) def getRowByFilename(self, filename): # Strip image:// prefix, trailing / suffix, and unquote... row = self.getRowByFilename_Impl(filename[8:-1], unquote=True) # Didn't find anything so try again, this time leave filename quoted, and don't truncate if not row: self.logger.log("Failed to find row by filename with the expected formatting, trying again (with prefix, quoted)") row = self.getRowByFilename_Impl(filename, unquote=False) return row def getRowByFilename_Impl(self, filename, unquote=True): if unquote: ufilename = MyUtility.normalise(filename) else: ufilename = filename # If string contains Unicode, replace Unicode chars with % and # use LIKE instead of equality if ufilename.encode("ascii", "ignore") == ufilename.encode("utf-8"): SQL = "WHERE url = \"%s\"" % ufilename else: self.logger.log("Removing ASCII from filename: [%s]" % ufilename) SQL = "WHERE url LIKE \"%s\"" % removeNonAscii(ufilename, "%") rows = self.getRows(filter=SQL, allfields=True) return rows[0] if rows != [] else None def removeNonAscii(self, s, replaceWith=""): if replaceWith == "": return "".join([x if ord(x) < 128 else ("%%%02x" % ord(x)) for x in s]) else: return "".join([x if ord(x) < 128 else replaceWith for x in s]) def dumpRow(self, row): line= ("%s%s%-14s%s%04d%s%04d%s%04d%s%19s%s%19s%s%s\n" % \ ((self.config.IDFORMAT % row["textureid"]), self.config.FSEP, row["cachedurl"], self.config.FSEP, row["sizes"][0]["height"], self.config.FSEP, row["sizes"][0]["width"], self.config.FSEP, row["sizes"][0]["usecount"], self.config.FSEP, row["sizes"][0]["lastused"], self.config.FSEP, row["lasthashcheck"], self.config.FSEP, row["url"])) self.logger.out(line) def getTextureFolders(self): # One extra folder (!) which is used to try and identify non-standard folders return ["0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f","!"] def getTextureFolderFilter(self, folder): if folder == "!": # Everything other than the regular folders return "WHERE (cachedurl < '0' or cachedurl > ':') and (cachedurl < 'a' or cachedurl > 'g')" else: return "WHERE cachedurl LIKE '%s/%%'" % folder # Raise this exception when we run out of replay log input class IOEndOfReplayLog(Exception): def __init__(self, value): self.value = value def __str__(self): return repr(self.value) # # Handle all JSON RPC communication. # # Uses sockets except for those methods (Files.*) that must # use HTTP. # class MyJSONComms(object): def __init__(self, config, logger, connecttimeout=None): self.config = config self.logger = logger self.connecttimeout = connecttimeout self.mysocket = None self.myweb = None self.WEB_LAST_STATUS = -1 self.WEB_LAST_REASON = "" self.config.WEB_SINGLESHOT = True self.aUpdateCount = self.vUpdateCount = 0 self.jcomms2 = None self.BUFFER_SIZE = 32768 self.QUIT_METHOD = self.QUIT_PARAMS = None self.LOG_REPLAYFILE = None def __enter__(self): return self def __exit__(self, atype, avalue, traceback): return def __del__(self): if self.mysocket: self.mysocket.close() if self.myweb: self.myweb.close() def close(self): pass def getSocket(self): lastexception = None if not self.mysocket: useipv = int(self.config.RPC_IPVERSION) if self.config.RPC_IPVERSION else None for ipversion in [socket.AF_INET6, socket.AF_INET]: if useipv and useipv == 4 and ipversion != socket.AF_INET: continue if useipv and useipv == 6 and ipversion != socket.AF_INET6: continue try: self.mysocket = socket.socket(ipversion, socket.SOCK_STREAM) self.mysocket.settimeout(self.connecttimeout) self.mysocket.connect((self.config.KODI_HOST, int(self.config.RPC_PORT))) self.mysocket.settimeout(None) self.logger.log("RPC connection established with IPv%s" % ("4" if ipversion == socket.AF_INET else "6")) self.config.RPC_IPVERSION = "4" if ipversion == socket.AF_INET else "6" return self.mysocket except Exception as e: lastexception = e pass else: raise lastexception if lastexception is not None else socket.error("Unknown socket error") return self.mysocket # Use a secondary socket object for simple lookups to avoid having to handle # re-entrant code due to notifications being received out of sequence etc. # Could instantiate an object whenever required, but keeping a reference here # should improve efficiency slightly. def getLookupObject(self): if not self.jcomms2: self.jcomms2 = MyJSONComms(self.config, self.logger) return self.jcomms2 def getWeb(self): if not self.myweb or self.config.WEB_SINGLESHOT: if self.myweb: self.myweb.close() self.myweb = httplib.HTTPConnection("%s:%s" % (self.config.KODI_HOST, self.config.WEB_PORT), timeout=self.connecttimeout) self.WEB_LAST_STATUS = -1 self.WEB_LAST_REASON = "" if self.config.DEBUG: self.myweb.set_debuglevel(1) return self.myweb def logreplay_open(self): try: thread = threading.current_thread().name if thread not in self.config.log_replay_fmap: self.LOG_REPLAYFILE = codecs.open(self.config.LOG_REPLAY_FILENAME, "r", encoding="utf-8") self.config.log_replay_fmap[thread] = self.LOG_REPLAYFILE else: self.LOG_REPLAYFILE = self.config.log_replay_fmap[thread] self.logreplay_mapthread(None) except: self.logger.err("ERROR: Unable to open replay file [%s] - exiting" % self.config.LOG_REPLAY_FILENAME, newLine=True, log=True) sys.exit(2) # When replaying muti-threaded input, map each new thread of input data to the # currently available threads. Once a thread is allocated a thread from the data, # it will only process data for that thread. def logreplay_mapthread(self, map_to_thread=None): thread = threading.current_thread().name if map_to_thread is None: # process data for all threads tpattern = thread if thread == "MainThread" else "Thread-[0-9]*" else: # process data for a specific thread tpattern = map_to_thread self.config.log_replay_tmap[map_to_thread] = thread self.web_re_result = re.compile("^.*:(%s)[ ]*: .*\.RECEIVED WEB DATA: ([0-9]*), (.*), (.*)$" % tpattern) self.json_re_result = re.compile("^.*:(%s)[ ]*: .*\.PARSING JSON DATA: (.*)$" % tpattern) # If the thread data being processed is mapped to a thread other than the current # thread then ignore this thread data def logreplay_ignore_thread(self, map_to_thread): with lock: thread = threading.current_thread().name if map_to_thread in self.config.log_replay_tmap: return self.config.log_replay_tmap[map_to_thread] != thread self.logreplay_mapthread(map_to_thread) return False # Read responses from log, use in place of actual socket/web responses (requests are never made) def logreplay(self, request, useWebServer): if not self.LOG_REPLAYFILE: self.logreplay_open() while True: line = self.LOG_REPLAYFILE.readline() if not line: break # Remove trailing newline and - if present, ie. Windows/DOS file format - carriage return if len(line) != 0 and line[-1] in ["\n", "\r"]: line = line[0:-1] if len(line) != 0 and line[-1] in ["\n", "\r"]: line = line[0:-1] if useWebServer: match = self.web_re_result.match(line) if match and not self.logreplay_ignore_thread(match.group(1)): self.WEB_LAST_STATUS = int(match.group(2)) self.WEB_LAST_REASON = match.group(3) return match.group(4).encode("utf-8") else: match = self.json_re_result.match(line) if match and not self.logreplay_ignore_thread(match.group(1)): return match.group(2).encode("utf-8") self.LOG_REPLAYFILE.close() raise IOEndOfReplayLog("End of replay log data") def sendWeb(self, request_type, url, id, request=None, headers={}, readAmount=0, timeout=15.0, rawData=False, domain=None, useSSL=False): if request is not None: sdata = json.dumps(request) self.logger.log("%s.JSON WEB REQUEST: [%s] [%s]" % (id, request_type, sdata)) else: sdata = None if domain is not None: self.logger.log("%s.DIRECT WEB REQUEST: [%s], HTTPS=%s, [%s] [%s]" % (id, request_type, useSSL, domain, url)) else: self.logger.log("%s.DIRECT WEB REQUEST: [%s], [%s]" % (id, request_type, url)) if self.config.LOG_REPLAY_FILENAME: data = self.logreplay(url, True) data = "" if sdata and data == "" else data if MyUtility.isPython3 and not rawData: data = data.decode("utf-8") else: if domain: if useSSL: web = httplib.HTTPSConnection(domain) else: web = httplib.HTTPConnection(domain) else: if self.config.WEB_AUTH_TOKEN: headers.update({"Authorization": "Basic %s" % self.config.WEB_AUTH_TOKEN}) web = self.getWeb() web.request(request_type, url, sdata, headers) if timeout is None: web.sock.setblocking(1) else: web.sock.settimeout(timeout) data = "" try: response = web.getresponse() self.WEB_LAST_STATUS = response.status self.WEB_LAST_REASON = response.reason if self.WEB_LAST_STATUS == httplib.UNAUTHORIZED: raise httplib.HTTPException("Remote web host requires webserver.username/webserver.password properties") if MyUtility.isPython3 and not rawData: if readAmount == 0: data = response.read().decode("utf-8") else: data = response.read(readAmount).decode("utf-8") else: if readAmount == 0: data = response.read() else: data = response.read(readAmount) except socket.timeout: self.logger.log("** iotimeout occurred during web request **") self.WEB_LAST_STATUS = httplib.REQUEST_TIMEOUT self.WEB_LAST_REASON = "Request Timeout" self.myweb.close() self.myweb = None data = "" except: if domain: self.logger.log("%s.RECEIVED WEB DATA: %d, %s, " % (id, response.status, response.reason), maxLen=256) raise if self.config.WEB_SINGLESHOT == False: self.logger.log("SWITCHING TO WEBSERVER.SINGLESHOT MODE") self.config.WEB_SINGLESHOT = True data = self.sendWeb(request_type, url, id, request, headers, readAmount, timeout, rawData) else: raise finally: if domain and web: web.close() if self.logger.LOGGING: if rawData: self.logger.log("%s.RECEIVED WEB DATA: %d, %s, " % (id, self.WEB_LAST_STATUS, self.WEB_LAST_REASON), maxLen=256) else: self.logger.log("%s.RECEIVED WEB DATA: %d, %s, %s" % (id, self.WEB_LAST_STATUS, self.WEB_LAST_REASON, data), maxLen=256) if sdata: return json.loads(data) if data != "" else "" else: return data def sendJSON(self, request, id, callback=None, timeout=5.0, checkResult=True, useWebServer=False, ignoreSocketError=False): request["jsonrpc"] = "2.0" request["id"] = id # Suppress complaints about Sets having no results (due to Sets not having been defined) if request["method"] == "VideoLibrary.GetMovieSets": checkResult=False # Following methods don't work over sockets - by design. if request["method"] in ["Files.PrepareDownload", "Files.Download"] or useWebServer: return self.sendWeb("POST", "/jsonrpc", id, request, {"Content-Type": "application/json"}, timeout=timeout) self.logger.log("%s.JSON SOCKET REQUEST:" % id, jsonrequest=request) START_IO_TIME = time.time() if self.config.LOG_REPLAY_FILENAME: jsocket = None else: jsocket = self.getSocket() if MyUtility.isPython3: jsocket.send(bytes(json.dumps(request), "utf-8")) else: jsocket.send(json.dumps(request)) ENDOFDATA = True LASTIO = 0 jdata = {} cbjdata = None while True: if ENDOFDATA: ENDOFDATA = False data = b"" if jsocket: jsocket.setblocking(1) try: if jsocket: newdata = jsocket.recv(self.BUFFER_SIZE) if len(data) == 0: jsocket.settimeout(1.0) else: newdata = self.logreplay(request, useWebServer) data += newdata LASTIO = time.time() self.logger.log("%s.BUFFER RECEIVED (len %d)" % (id, len(newdata))) if len(newdata) == 0: raise IOError("nodata") READ_ERR = False except (IOError, IOEndOfReplayLog) as e: # Hack to exit monitor mode when socket dies if callback: jdata = {"jsonrpc":"2.0","method":"System.OnQuit","params":{"data":-1,"sender":"xbmc"}} self.handleResponse(id, jdata, callback) return jdata elif ignoreSocketError == False: self.logger.err("ERROR: Socket closed prematurely - exiting", newLine=True, log=True) sys.exit(2) else: return {} except socket.error as e: READ_ERR = True # Keep reading unless accumulated data is a likely candidate for successful parsing... if not READ_ERR and len(data) != 0 and (data[-1:] == b"}" or data[-2:] == b"}\n"): # If data is not a str (Python2) then decode Python3 bytes to Unicode representation if isinstance(data, str): udata = MyUtility.toUnicode(data) else: try: udata = data.decode("utf-8") except UnicodeDecodeError as e: udata = data try: START_PARSE_TIME = time.time() # Parse messages, to ensure the entire buffer is valid # If buffer is not valid (VlueError exception), we may need to read more data. messages = [] for m in self.parseResponse(udata): if self.logger.LOGGING and messages == []: if udata.find("\n") != -1: self.logger.log2("%s.PARSING JSON DATA: " % id, udata.replace("\t", "").replace("\n", ""), maxLen=256) else: self.logger.log2("%s.PARSING JSON DATA: " % id, udata, maxLen=256) messages.append(m) # Discard these buffers which could potentially be very large, as they're no longer required del data, udata self.logger.log("%s.PARSING COMPLETE, elapsed time: %f seconds" % (id, time.time() - START_PARSE_TIME)) # Process any notifications first. # Any message with an id must be processed after notification - should only be one at most... result = False jdata = {} for m in messages: if "id" not in m: if callback: if self.handleResponse(id, m, callback): result = True elif self.logger.LOGGING: self.logger.log("%s.IGNORING NOTIFICATION" % id, jsonrequest=m, maxLen=256) elif m["id"] == id: jdata = m # Discard - no longer required del messages # "result" on response for an Application.SetMute()/SetVolume() is # not iterable so just ignore it if we cause an exception... try: if ("result" in jdata and "limits" in jdata["result"]): self.logger.log("%s.RECEIVED LIMITS: %s" % (id, jdata["result"]["limits"])) except TypeError: pass # Flag to reset buffers next time we read the socket. ENDOFDATA = True # callback result for a commingled Notification - stop blocking/reading and # return to caller with response (jdata) if result: break # Got a response... if jdata != {}: # If callback defined, pass it the message then break if result is True. # Otherwise break only if message has an id, that is to # say, continue reading data (blocking) until a message (response) # with an id is available. if callback: if self.handleResponse(id, jdata, callback): break #Save this jdata as it's our original response if cbjdata is None: cbjdata = jdata elif "id" in jdata: break if callback: self.logger.log("%s.READING SOCKET UNTIL CALLBACK SUCCEEDS..." % id) else: self.logger.log("%s.READING SOCKET FOR A RESPONSE..." % id) except ValueError as e: # If we think we've reached EOF (no more data) and we have invalid data then # raise exception, otherwise continue reading more data if READ_ERR: self.logger.log("%s.VALUE ERROR EXCEPTION: %s" % (id, str(e))) raise else: self.logger.log("%s.Incomplete JSON data - continue reading socket" % id) if self.logger.VERBOSE: self.logger.log2("Ignored Value Error: ", str(e)) continue except Exception as e: self.logger.log("%s.GENERAL EXCEPTION: %s" % (id, str(e))) raise # Still more data to be read... if not ENDOFDATA: if (time.time() - LASTIO) > timeout: self.logger.log("SOCKET IO TIMEOUT EXCEEDED") raise socket.error("Socket IO timeout exceeded") if cbjdata is not None: jdata = cbjdata if checkResult and not "result" in jdata: self.logger.out("%s.ERROR: JSON response has no result!\n%s\n" % (id, jdata)) self.logger.log("%s.FINISHED, elapsed time: %f seconds" % (id, time.time() - START_IO_TIME)) return jdata # Split data into individual JSON objects. def parseResponse(self, data): decoder = json._default_decoder _w=json.decoder.WHITESPACE.match idx = _w(data, 0).end() end = len(data) try: while idx != end: (val, idx) = decoder.raw_decode(data, idx=idx) yield val idx = _w(data, idx).end() except ValueError as exc: # raise ValueError("%s (%r at position %d)." % (exc, data[idx:], idx)) raise ValueError("%s" % exc) # Process Notifications, optionally executing a callback function for # additional custom processing. def handleResponse(self, callingId, jdata, callback): if "error" in jdata: return True id = jdata["id"] if "id" in jdata else None method = jdata["method"] if "method" in jdata else jdata.get("result", None) params = jdata["params"] if "params" in jdata else None if callback: cname = callback.__name__ self.logger.log("%s.PERFORMING CALLBACK: Name [%s], with Id [%s], Method [%s], Params [%s]" % (callingId, cname, id, method, params)) result = callback(id, method, params) self.logger.log("%s.CALLBACK RESULT: [%s] Name [%s], Id [%s], Method [%s], Params [%s]" % (callingId, result, cname, id, method, params)) if result: self.QUIT_METHOD = method self.QUIT_PARAMS = params return result return False def listen(self): REQUEST = {"method": "JSONRPC.Ping"} self.sendJSON(REQUEST, "libListen", callback=self.speak, checkResult=False) def speak(self,id, method, params): # Only interested in Notifications... if id: return False item = None title = None pmsg = None if params["data"]: pmsg = json.dumps(params["data"]) if type(params["data"]) is dict: if "item" in params["data"]: item = params["data"]["item"] elif "type" in params["data"]: item = params["data"] if item: title = self.getTitleForLibraryItem(item.get("type", None), item.get("id", None)) if not pmsg: pmsg = "{}" if title: self.logger.out("%s: %-21s: %s [%s]" % (datetime.datetime.now(), method, pmsg, title), newLine=True) else: self.logger.out("%s: %-21s: %s" % (datetime.datetime.now(), method, pmsg), newLine=True) return True if method == "System.OnQuit" else False def jsonWaitForScanFinished(self, id, method, params): if method.endswith("Library.OnUpdate") and "data" in params: if method == "AudioLibrary.OnUpdate": self.aUpdateCount += 1 if method == "VideoLibrary.OnUpdate": self.vUpdateCount += 1 if "item" in params["data"]: item = params["data"]["item"] elif "type" in params["data"]: item = params["data"] else: item = None if item: iType = item["type"] libraryId = item["id"] title = self.getTitleForLibraryItem(iType, libraryId) if title: self.logger.out("Updating library: New %-9s %5d [%s]\n" % (iType + "id", libraryId, title)) else: self.logger.out("Updating library: New %-9s %5d\n" % (iType + "id", libraryId)) return True if method.endswith("Library.OnScanFinished") else False def jsonWaitForCleanFinished(self, id, method, params): if method.endswith("Library.OnRemove") and "data" in params: if "item" in params["data"]: item = params["data"]["item"] elif "type" in params["data"]: item = params["data"] else: item = None if item: iType = item["type"] libraryId = item["id"] title = self.getTitleForLibraryItem(iType, libraryId) if title: self.logger.out("Cleaning library: %-9s %5d [%s]\n" % (iType + "id", libraryId, title)) else: self.logger.out("Cleaning library: %-9s %5d\n" % (iType + "id", libraryId)) return True if method.endswith("Library.OnCleanFinished") else False def addProperties(self, request, fields): if not "properties" in request["params"]: return aList = request["params"]["properties"] if fields is not None: for f in [f.strip() for f in fields.split(",")]: if f != "" and f not in aList: aList.append(f) request["params"]["properties"] = aList def delProperties(self, request, fields): if not "properties" in request["params"]: return aList = request["params"]["properties"] if fields is not None: for f in [f.strip() for f in fields.split(",")]: if f != "" and f in aList: aList.remove(f) request["params"]["properties"] = aList def addFilter(self, request, newFilter, condition="and"): filter = request["params"] if "filter" in filter: filter["filter"] = {condition: [filter["filter"], newFilter]} else: filter["filter"] = newFilter request["params"] = filter def rescanDirectories(self, workItems): if workItems == {}: return doRefresh = (gConfig.JSON_HAS_REFRESH_REFACTOR and not gConfig.QA_USEOLDREFRESHMETHOD) method = "Refresh" if doRefresh else "Remove" # Seems to be a bug in rescan method when scanning the root folder of a source # So if any items are in the root folder, just scan the entire library after # items have been removed. rootScan = False sources = self.getSources("video") for directory in sorted(workItems): (mediatype, dpath) = directory.split(";", 1) if dpath in sources: rootScan = True for directory in sorted(workItems): (mediatype, dpath) = directory.split(";", 1) for disc_folder in [".BDMV$", ".VIDEO_TS$"]: re_match = re.search(disc_folder, dpath, flags=re.IGNORECASE) if re_match: dpath = dpath[:re_match.start()] break if mediatype == "movies": scanMethod = "VideoLibrary.Scan" removeMethod = "VideoLibrary.%sMovie" % method idName = "movieid" elif mediatype == "tvshows": scanMethod = "VideoLibrary.Scan" removeMethod = "VideoLibrary.%sTVShow" % method idName = "tvshowid" elif mediatype == "episodes": scanMethod = "VideoLibrary.Scan" removeMethod = "VideoLibrary.%sEpisode" % method idName = "episodeid" else: raise ValueError("mediatype [%s] not yet implemented" % mediatype) for libraryitem in workItems[directory]: libraryid = libraryitem["id"] if doRefresh: self.logger.log("Refreshing %s %d in media library." % (idName, libraryid)) else: self.logger.log("Removing %s %d from media library." % (idName, libraryid)) REQUEST = {"method": removeMethod, "params":{idName: libraryid}} if doRefresh: REQUEST["params"]["ignorenfo"] = False if mediatype == "tvshows": REQUEST["params"]["refreshepisodes"] = False self.sendJSON(REQUEST, "lib%s" % method) if doRefresh: gLogger.out("Updating library: Refreshed %s %d [%s]" % (idName, libraryid, libraryitem["name"]), newLine=True) if not doRefresh and not rootScan: self.scanDirectory(scanMethod, path=dpath) if not doRefresh and rootScan: self.scanDirectory(scanMethod) def scanDirectory(self, scanMethod, path=None): if path and path != "": self.logger.out("Rescanning directory: %s..." % path, newLine=True, log=True) REQUEST = {"method": scanMethod, "params":{"directory": path}} else: self.logger.out("Rescanning library...", newLine=True, log=True) REQUEST = {"method": scanMethod} if self.config.JSON_HAS_LIB_SHOWDIALOGS_PARAM: if "params" not in REQUEST: REQUEST["params"] = {} REQUEST["params"].update({"showdialogs": self.config.SCAN_SHOW_DIALOGS}) self.sendJSON(REQUEST, "libRescan", callback=self.jsonWaitForScanFinished, checkResult=False) def cleanLibrary(self, cleanMethod): self.logger.out("Cleaning library...", newLine=True, log=True) REQUEST = {"method": cleanMethod} if self.config.JSON_HAS_LIB_SHOWDIALOGS_PARAM: REQUEST["params"] = {"showdialogs": self.config.CLEAN_SHOW_DIALOGS} self.sendJSON(REQUEST, "libClean", callback=self.jsonWaitForCleanFinished, checkResult=False) def getDirectoryList(self, path, mediatype="files", properties=["file","lastmodified"], use_cache=True, timestamp=False, honour_nomedia=False): data = MyUtility.getDirectoryCacheItem(properties, path) if not data: REQUEST = {"method":"Files.GetDirectory", "params": {"directory": path, "media": mediatype, "properties": properties}} data = self.sendJSON(REQUEST, "libDirectory", checkResult=False) # Fix null being returned for "files" on some systems... if "result" in data and "files" in data["result"]: if data["result"]["files"] is None: data["result"]["files"] = [] for f in data["result"]["files"]: if "file" in f: # Real directories won't have extensions, but .m3u and .pls playlists will # leading to infinite recursion, so fix the filetype so as not to try and # traverse playlists. if f["filetype"] == "directory": extension = os.path.splitext(f["file"])[1] if extension and extension.find("/") != -1 and extension.find("\\") != -1: f["filetype"] = "file" # Convert last modified date/time to epoch if timestamp: self.setTimeStamp(f) if use_cache: MyUtility.setDirectoryCacheItem(data, properties, path) if honour_nomedia and data and "result" in data and "files" in data["result"]: for f in data["result"]["files"]: if f["label"] == ".nomedia" or f["label"] == ".nomedia.": data["result"]["files"] = [] break return data def setTimeStamp(self, item): if "lastmodified" in item: try: item["lastmodified_timestamp"] = MyUtility.SinceEpoch(datetime.datetime.strptime(item["lastmodified"], self.config.qa_lastmodified_fmt)) except ValueError: self.logger.err("ERROR: Invalid \"lastmodified\" date detected - try specifying @modifieddate.mdy=%s" % ("no" if self.config.MDATE_MDY else "yes"), newLine=True) sys.exit(2) def getExtraArt(self, item): if not (item and self.config.CACHE_EXTRA): return [] # Movies, Tags and TV shows have a file property which can be used as the media root. # Artists and Albums do not, so try and find a usable local path from the # fanart/thumbnail artwork. directory = None if "file" in item: directory = MyUtility.unstackFiles(item["file"])[0] else: for a in ["fanart", "thumbnail"]: if a in item: tmp = MyUtility.normalise(item[a], strip=True) hostname = re.search("^.*@", tmp) if hostname and hostname.end() < 10: directory = tmp[hostname.end():] break elif not tmp.startswith("http:"): directory = tmp break if not directory: return [] # Remove filename, leaving just parent directory. # Could use os.path.dirname() here but we need # to know which slash is being used so that it # can be appended before the relevant subdir is added. for slash in ["/", "\\"]: pos = directory.rfind(slash) if pos != -1: directory = "%s" % directory[:pos+1] SLASH = directory[pos:pos+1] break else: return [] data = self.getDirectoryList(directory) if "result" not in data: return [] if "files" not in data["result"]: return [] artitems = [] if self.config.CACHE_EXTRA_FANART: artitems.append("%sextrafanart%s" % (SLASH, SLASH)) if self.config.CACHE_EXTRA_THUMBS: artitems.append("%sextrathumbs%s" % (SLASH, SLASH)) if self.config.CACHE_VIDEO_EXTRAS: artitems.append("%sextras%s" % (SLASH, SLASH)) artitems.append("%sExtras%s" % (SLASH, SLASH)) dirs = [] for file in data["result"]["files"]: if file["filetype"] == "directory" and file["file"]: for a in artitems: if file["file"].endswith(a): dirs.append({"file": file["file"], "type": a[1:-1]}) break files = [] for dir in dirs: data = self.getDirectoryList(dir["file"]) if "result" in data and "files" in data["result"]: for file in data["result"]["files"]: if file["filetype"] == "file" and file["file"]: if os.path.splitext(file["file"])[1].lower() in [".jpg", ".png", ".tbn"]: files.append({"file": MyUtility.denormalise(file["file"], prefix=True), "type": dir["type"].lower()}) return files def getSeasonAll(self, filename): # If "Season All" items are not being cached, return no results if self.config.CACHE_HIDEALLITEMS: return (None, None, None) # Not able to get a directory for remote files... if filename.find("image://http") != -1: return (None, None, None) directory = MyUtility.normalise(filename, strip=True) # Remove filename, leaving just parent directory. # Could use os.path.dirname() here but we need to know # which slash is being used so that it can be # appended before the filename is added by the caller. for slash in ["/", "\\"]: pos = directory.rfind(slash) if pos != -1: directory = directory[:pos] break else: return (None, None, None) data = self.getDirectoryList(directory) if "result" in data and "files" in data["result"]: poster_url = fanart_url = banner_url = None for f in data["result"]["files"]: if f["filetype"] == "file": fname = os.path.split(f["label"])[1].lower() if fname.startswith("season-all-poster."): poster_url = MyUtility.joinQuotedPath(filename, f["file"]) elif not poster_url and fname.startswith("season-all."): poster_url = MyUtility.joinQuotedPath(filename, f["file"]) elif fname.startswith("season-all-banner."): banner_url = MyUtility.joinQuotedPath(filename, f["file"]) elif fname.startswith("season-all-fanart."): fanart_url = MyUtility.joinQuotedPath(filename, f["file"]) return (poster_url, fanart_url, banner_url) return (None, None, None) def getDownloadURL(self, filename): REQUEST = {"method":"Files.PrepareDownload", "params":{"path": filename}} data = self.sendJSON(REQUEST, "preparedl") if "result" in data: return "/%s" % data["result"]["details"]["path"] else: return None # Get file details from a directory lookup, this prevents errors on Kodi when # the file doesn't exist (unless the directory doesn't exist), and also allows the # query results to be cached for use by subsequent file requests in the same directory. def getFileDetails(self, filename, properties=["file", "lastmodified", "size"]): data = self.getDirectoryList(os.path.dirname(filename), mediatype="files", properties=properties) if "result" in data: files = data["result"].get("files", []) for file in [x for x in files if x["filetype"] == "file" and x.get("file", None) == filename]: if "lastmodified" in properties: self.setTimeStamp(file) return file return None # Get title of item - usually during a notification. As this can be # an OnRemove notification, don't check for result as the item may have # been removed before it can be looked up, in which case return None. def getTitleForLibraryItem(self, iType, libraryId): title = None if iType and libraryId: # Use the secondary socket object to avoid consuming # notifications that are meant for the caller. if iType == "song": title = self.getLookupObject().getSongName(libraryId) elif iType == "movie": title = self.getLookupObject().getMovieName(libraryId) elif iType == "tvshow": title = self.getLookupObject().getTVShowName(libraryId) elif iType == "episode": title = self.getLookupObject().getEpisodeName(libraryId) elif iType == "musicvideo": title = self.getLookupObject().getMusicVideoName(libraryId) return title def getSongName(self, songid): REQUEST = {"method":"AudioLibrary.GetSongDetails", "params":{"songid": songid, "properties":["title", "artist", "albumartist"]}} data = self.sendJSON(REQUEST, "libSong", checkResult=False) if "result" in data and "songdetails" in data["result"]: s = data["result"]["songdetails"] if s["artist"]: return "%s (%s)" % (s["title"], "/".join(s["artist"])) else: return "%s (%s)" % (s["title"], "/".join(s["albumartist"])) else: return None def getTVShowName(self, tvshowid): REQUEST = {"method":"VideoLibrary.GetTVShowDetails", "params":{"tvshowid": tvshowid, "properties":["title"]}} data = self.sendJSON(REQUEST, "libTVShow", checkResult=False) if "result" in data and "tvshowdetails" in data["result"]: t = data["result"]["tvshowdetails"] return "%s" % t["title"] else: return None def getEpisodeName(self, episodeid): REQUEST = {"method":"VideoLibrary.GetEpisodeDetails", "params":{"episodeid": episodeid, "properties":["title", "showtitle", "season", "episode"]}} data = self.sendJSON(REQUEST, "libEpisode", checkResult=False) if "result" in data and "episodedetails" in data["result"]: e = data["result"]["episodedetails"] return "%s S%02dE%02d (%s)" % (e["showtitle"], e["season"], e["episode"], e["title"]) else: return None def getMovieName(self, movieid): REQUEST = {"method":"VideoLibrary.GetMovieDetails", "params":{"movieid": movieid, "properties":["title"]}} data = self.sendJSON(REQUEST, "libMovie", checkResult=False) if "result" in data and "moviedetails" in data["result"]: m = data["result"]["moviedetails"] return "%s" % m["title"] else: return None def getMusicVideoName(self, videoid): REQUEST = {"method":"VideoLibrary.GetMusicVideoDetails", "params":{"musicvideoid": videoid, "properties":["title"]}} data = self.sendJSON(REQUEST, "libMusicVideo", checkResult=False) if "result" in data and "musicvideodetails" in data["result"]: m = data["result"]["musicvideodetails"] return "%s" % m["title"] else: return None def removeLibraryItem(self, iType, libraryId): if iType and libraryId: # Use the secondary socket object to avoid consuming # notifications that are meant for the caller. if iType == "movie": (method, arg) = ("VideoLibrary.RemoveMovie", "movieid") elif iType == "tvshow": (method, arg) = ("VideoLibrary.RemoveTVShow", "tvshowid") elif iType == "episode": (method, arg) = ("VideoLibrary.RemoveEpisode", "episodeid") elif iType == "musicvideo": (method, arg) = ("VideoLibrary.RemoveMusicVideo", "musicvideoid") REQUEST = {"method": method, "params":{arg: libraryId}} data = self.sendJSON(REQUEST, "libRemove", checkResult=True) def dumpJSON(self, data, decode=False, ensure_ascii=True): if decode: self.logger.progress("Decoding URLs...") self.unquoteArtwork(data) self.logger.progress("") self.logger.out(json.dumps(data, indent=2, ensure_ascii=ensure_ascii, sort_keys=True), newLine=True) def unquoteArtwork(self, items): for item in items: for field in item: if field in ["seasons", "episodes", "channels", "tc.members"]: self.unquoteArtwork(item[field]) elif field in ["file", "fanart", "thumbnail"]: item[field] = MyUtility.normalise(item[field]) elif field == "art": art = item["art"] for image in art: art[image] = MyUtility.normalise(art[image]) elif field == "cast": for cast in item["cast"]: if "thumbnail" in cast: cast["thumbnail"] = MyUtility.normalise(cast["thumbnail"]) def getSources(self, media, labelPrefix=False, withLabel=None): REQUEST = {"method": "Files.GetSources", "params":{"media": media}} data = self.sendJSON(REQUEST, "libSources") source_list = [] if "result" in data and "sources" in data["result"]: for source in data["result"]["sources"]: file = source["file"] label = source["label"] if not withLabel or withLabel.lower() == label.lower(): if file.startswith("multipath://"): for apath in file[12:].split("/"): if apath != "": apath= MyUtility.normalise(apath)[:-1] if labelPrefix: source_list.append("%s: %s" % (label, apath)) else: source_list.append(apath) else: apath = MyUtility.normalise(file)[:-1] if labelPrefix: source_list.append("%s: %s" % (label, apath)) else: source_list.append(apath) return sorted(source_list) def getAllFilesForSource(self, mediatype, labels, ignore_list=None, honour_nomedia=False): if mediatype == "songs": mtype = "music" includeList = self.config.audio_filetypes elif mediatype == "pictures": mtype = "pictures" includeList = self.config.picture_filetypes else: mtype = "video" includeList = self.config.video_filetypes if self.config.IGNORE_PLAYLISTS: for ignoreExt in [".m3u", ".pls", ".cue"]: if ignoreExt in includeList: includeList.remove(ignoreExt) fileList = [] for label in labels: sources = self.getSources(mtype, withLabel=label) for path in sources: self.logger.progress("Walking source: %s" % path) for file in self.getFiles(path, honour_nomedia): ext = os.path.splitext(file)[1].lower() if ext not in includeList: continue if os.path.splitext(file)[0].lower().endswith("trailer"): continue isVIDEOTS = (file.find("/VIDEO_TS/") != -1 or file.find("\\VIDEO_TS\\") != -1) isBDMV = (file.find("/BDMV/") != -1 or file.find("\\BDMV\\") != -1) if isVIDEOTS and ext != ".vob": continue if isBDMV and ext != ".m2ts": continue if ignore_list is not None: ignore_matched = False for ignore in ignore_list: if ignore.search(os.path.basename(file)): ignore_matched = True break if ignore_matched: continue # Avoid adding file to list more than once, which is possible # if a folder appears within multiple different sources, or the # same source is processed more than once... if not file in fileList: fileList.append(file) if fileList == []: self.logger.out("WARNING: no files obtained from filesystem - ensure valid source(s) specified!", newLine=True) fileList.sort() return fileList def getFiles(self, path, honour_nomedia=False): fileList = [] self.getFilesForPath(fileList, path, honour_nomedia) return fileList def getFilesForPath(self, fileList, path, honour_nomedia=False): data = self.getDirectoryList(path, use_cache=False, honour_nomedia=honour_nomedia) if not "result" in data: return if not "files" in data["result"]: return for file in data["result"]["files"]: if "file" in file: if file["filetype"] == "directory": self.getFilesForPath(fileList, os.path.dirname(file["file"]), honour_nomedia) else: fileList.append(file["file"]) def setPower(self, state): if state == "exit": REQUEST = {"method": "Application.Quit"} else: REQUEST = {"method": "System.%s" % state.capitalize()} data = self.sendJSON(REQUEST, "libPower") def getData(self, action, mediatype, filter = None, useExtraFields = False, secondaryFields = None, tvshow = None, tvseason = None, channelgroupid = None, lastRun = False, subType = None, uniquecast = None): EXTRA = mediatype SECTION = mediatype FILTER = "title" TITLE = "title" IDENTIFIER = "%sid" % re.sub("(.*)s$", "\\1", mediatype) if mediatype == "addons": REQUEST = {"method":"Addons.GetAddons", "params":{"properties":["name", "version", "thumbnail", "fanart", "path"]}} FILTER = "name" TITLE = "name" elif mediatype in ["pvr.tv", "pvr.radio"]: REQUEST = {"method":"PVR.GetChannelGroups", "params":{"channeltype": mediatype.split(".")[1]}} SECTION = "channelgroups" FILTER = "channeltype" TITLE = "label" IDENTIFIER = "channelgroupid" elif mediatype in ["pvr.tv.channel", "pvr.radio.channel"]: REQUEST = {"method":"PVR.GetChannels", "params":{"channelgroupid": channelgroupid, "properties": ["channeltype", "channel", "thumbnail", "hidden", "locked", "lastplayed"]}} SECTION = "channels" FILTER = "channel" TITLE = "channel" IDENTIFIER = "channelid" elif mediatype == "albums": REQUEST = {"method":"AudioLibrary.GetAlbums", "params":{"sort": {"order": "ascending", "method": "label"}, "properties":["title", "artist", "fanart", "thumbnail"]}} FILTER = "album" elif mediatype == "artists": REQUEST = {"method":"AudioLibrary.GetArtists", "params":{"sort": {"order": "ascending", "method": "artist"}, "albumartistsonly": False, "properties":["fanart", "thumbnail"]}} FILTER = "artist" TITLE = "artist" elif mediatype == "songs": REQUEST = {"method":"AudioLibrary.GetSongs", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "track", "artist", "album", "fanart", "thumbnail"]}} elif mediatype == "song-members": REQUEST = {"method":"AudioLibrary.GetSongs", "params":{"sort": {"order": "ascending", "method": "track"}, "properties":["title", "track", "album", "fanart", "thumbnail", "file"]}} EXTRA = "songs" SECTION = "songs" IDENTIFIER = "songid" elif mediatype == "musicvideos": REQUEST = {"method":"VideoLibrary.GetMusicVideos", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art", "tag"]}} EXTRA = "musicvideos" SECTION = "musicvideos" IDENTIFIER = "musicvideoid" elif mediatype in ["movies", "tags"]: REQUEST = {"method":"VideoLibrary.GetMovies", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}} if mediatype == "tags": REQUEST["params"]["properties"].append("tag") EXTRA = "movies" SECTION = "movies" IDENTIFIER = "movieid" elif mediatype == "sets": REQUEST = {"method":"VideoLibrary.GetMovieSets", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}} FILTER = "" elif mediatype == "sets-members": REQUEST = {"method":"VideoLibrary.GetMovies", "params":{"sort": {"order": "ascending", "method": "sorttitle"}, "properties":["file", "set", "title", "sorttitle"]}} EXTRA = "movies" SECTION = "movies" IDENTIFIER = "movieid" elif mediatype == "tvshows": REQUEST = {"method":"VideoLibrary.GetTVShows", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}} EXTRA = "tvshows.tvshow" elif mediatype == "seasons": REQUEST = {"method":"VideoLibrary.GetSeasons", "params":{"sort": {"order": "ascending", "method": "season"}, "tvshowid": tvshow["tvshowid"], "properties":["season", "art"]}} FILTER = "" TITLE = "label" EXTRA = "tvshows.season" IDENTIFIER = "season" elif mediatype == "episodes": REQUEST = {"method":"VideoLibrary.GetEpisodes", "params":{"sort": {"order": "ascending", "method": "label"}, "tvshowid": tvshow["tvshowid"], "season": tvseason["season"], "properties":["art"]}} FILTER = "" TITLE = "label" EXTRA = "tvshows.episode" elif mediatype == "agenres": REQUEST = {"method":"AudioLibrary.GetGenres", "params":{"properties":["title", "thumbnail"]}} FILTER = "title" TITLE = "title" SECTION = "genres" IDENTIFIER = "genreid" elif mediatype == "vgenres": REQUEST = {"method":"VideoLibrary.GetGenres", "params":{"type": subType, "properties":["title", "thumbnail"]}} FILTER = "title" TITLE = "title" SECTION = "genres" IDENTIFIER = "genreid" else: raise ValueError("Invalid mediatype: [%s]" % mediatype) if mediatype == "tags": if not filter or filter.strip() == "": self.addFilter(REQUEST, {"field": "tag", "operator": "contains", "value": "%"}) else: word = 0 filterBoolean = "and" for tag in [x.strip() for x in re.split("( and | or )", filter)]: word += 1 if (word%2 == 0) and tag in ["and","or"]: filterBoolean = tag else: self.addFilter(REQUEST, {"field": "tag", "operator": "contains", "value": tag}, filterBoolean) elif mediatype in ["sets-members", "song-members"]: _f = "set" if mediatype == "sets-members" else "album" if filter and not self.config.FILTER_FIELD: self.addFilter(REQUEST, {"field": _f, "operator": self.config.FILTER_OPERATOR, "value": filter}) else: # JSON filter is broken when handling empty (null) strings - they're ignored - though # hopefully this will be fixed in a later version of API, in which case use it if self.config.JSON_HAS_FILTERNULLVALUE: self.addFilter(REQUEST, {"field": _f, "operator": "isnot", "value": ""}) else: self.addFilter(REQUEST, {"field": _f, "operator": "doesnotcontain", "value": "@@@@@@@@@@@@"}) elif filter and filter.strip() != "" and not mediatype in ["addons", "agenres", "vgenres", "sets", "seasons", "episodes", "pvr.tv", "pvr.radio", "pvr.channels"]: FILTER = self.config.FILTER_FIELD if self.config.FILTER_FIELD else FILTER self.addFilter(REQUEST, {"field": FILTER, "operator": self.config.FILTER_OPERATOR, "value": filter}) if mediatype in ["movies", "tags", "episodes"]: if lastRun and self.config.LASTRUNFILE_DATETIME: self.addFilter(REQUEST, {"field": "dateadded", "operator": "after", "value": self.config.LASTRUNFILE_DATETIME}) # Add extra required fields/properties based on action to be performed if action == "duplicates": if "art" in REQUEST["params"]["properties"]: REQUEST["params"]["properties"].remove("art") self.addProperties(REQUEST, "file") self.addProperties(REQUEST, "imdbnumber") self.addProperties(REQUEST, "playcount") self.addProperties(REQUEST, "lastplayed") self.addProperties(REQUEST, "dateadded") elif action == "imdb": if self.config.IMDB_PERIOD_FROM and mediatype in ["movies", "episodes"]: self.addFilter(REQUEST, {"field": "dateadded", "operator": "after", "value": self.config.IMDB_PERIOD_FROM}) if mediatype == "movies": self.addProperties(REQUEST, "imdbnumber") self.addProperties(REQUEST, ",".join(self.config.IMDB_FIELDS_MOVIES)) elif mediatype in ["tvshows", "episodes"]: if mediatype == "tvshows": self.addProperties(REQUEST, "year") elif mediatype == "episodes": self.delProperties(REQUEST, "genre") self.addProperties(REQUEST, "file") self.addProperties(REQUEST, ",".join(self.config.IMDB_FIELDS_TVSHOWS)) elif action == "missing": for unwanted in ["artist", "art", "fanart", "thumbnail"]: if unwanted in REQUEST["params"]["properties"]: REQUEST["params"]["properties"].remove(unwanted) if mediatype in ["songs", "movies", "tvshows", "episodes"]: self.addProperties(REQUEST, "file") elif action == "watched" and mediatype in ["movies", "episodes"]: if "art" in REQUEST["params"]["properties"]: REQUEST["params"]["properties"].remove("art") if mediatype == "movies": self.addProperties(REQUEST, "year") self.addProperties(REQUEST, "playcount, lastplayed, resume") elif action == "qa": if self.config.QADATE and mediatype in ["movies", "tags", "episodes"]: self.addFilter(REQUEST, {"field": "dateadded", "operator": "after", "value": self.config.QADATE}) if mediatype in ["songs", "movies", "tags", "tvshows", "episodes"]: self.addProperties(REQUEST, "file") self.addProperties(REQUEST, ", ".join(self.config.getQAFields("zero", EXTRA))) self.addProperties(REQUEST, ", ".join(self.config.getQAFields("blank", EXTRA))) elif action == "dump": if mediatype in ["songs", "movies", "tvshows", "episodes", "musicvideos"]: self.addProperties(REQUEST, "file") if "extrajson.%s" % EXTRA in self.config.XTRAJSON: extraFields = self.config.XTRAJSON["extrajson.%s" % EXTRA] if EXTRA != "" else None else: extraFields = None if useExtraFields and extraFields: self.addProperties(REQUEST, extraFields) if secondaryFields: self.addProperties(REQUEST, secondaryFields) elif action == "query" and not mediatype in ["tvshows", "seasons", "pvr.tv", "pvr.radio"]: if secondaryFields: self.addProperties(REQUEST, secondaryFields) elif action == "cache": if mediatype in ["movies", "tags", "tvshows", "episodes"] and self.config.CACHE_CAST_THUMB: self.addProperties(REQUEST, "cast") if self.config.CACHE_EXTRA: if mediatype in ["movies", "tags", "tvshows"]: self.addProperties(REQUEST, "file") elif mediatype in ["artists", "albums"]: self.addProperties(REQUEST, "fanart") self.addProperties(REQUEST, "thumbnail") return (SECTION, TITLE, IDENTIFIER, self.getDataProxy(mediatype, REQUEST, trim_cast_thumbs=(action != "dump"), uniquecast=uniquecast)) # Load data chunked, or in one single query. # TV Shows, seasons and episodes are already "chunked" by definition. # If specified, remove cast members without thumbnails to reduce memory footprint. def getDataProxy(self, mediatype, request, trim_cast_thumbs=True, idname=None, uniquecast=None): if not idname: idname = "lib%s" % mediatype.capitalize() mediatype = mediatype.lower() if self.config.CHUNKED: silent = (mediatype in ["tvshows", "seasons", "episodes"]) data = self.chunkedLoad(mediatype, request, trim_cast_thumbs, idname=idname, silent=silent, uniquecast=uniquecast) else: data = self.sendJSON(request, idname) if "result" in data and trim_cast_thumbs and "cast" in request.get("params",{}).get("properties",[]): for section in data["result"]: if section != "limits": for item in data["result"][section]: self.removecastwithoutthumbs(item, uniquecast) return data # Load library data in chunks, using limits. # Return resulting list of all requested items. def chunkedLoad(self, mediatype, request, trim_cast_thumbs=True, idname=None, silent=False, uniquecast=None): if not idname: idname = "libChunked%s" % mediatype.capitalize() CHUNK_SIZE = 400 if mediatype in ["movies", "tags", "sets-members", "tvshows"]: if "cast" in request.get("params",{}).get("properties",[]): CHUNK_SIZE = 35 chunk = 0 chunk_start = 0 total_items = 1 chunks = 0 section = None results = [] while chunk_start < total_items: chunk += 1 if not silent: # Don't yet know how many chunks there will be if chunk_start != 0: self.logger.progress("Loading %s: Chunk %d of %d..." % (mediatype.capitalize(), chunk, chunks)) else: self.logger.progress("Loading %s: Chunk %d..." % (mediatype.capitalize(), chunk)) request["params"]["limits"] = {"start": chunk_start, "end": chunk_start + CHUNK_SIZE} data = self.sendJSON(request, idname) if "result" not in data: break #Get total_items and section name once first chunk is retrieved if chunk_start == 0: if "limits" not in data["result"]: break total_items = data["result"]["limits"]["total"] chunks = -(-total_items // CHUNK_SIZE) self.logger.log("Chunk processing: found %d %s, retrieving in chunks of %d" % (total_items, mediatype, CHUNK_SIZE)) for s in data.get("result", {}): if s != "limits": section = s break else: break # Add section to accumulated results if section and section in data["result"]: # Remove those cast members without thumbnails if trim_cast_thumbs and "cast" in request.get("params",{}).get("properties",[]): for item in data["result"][section]: self.removecastwithoutthumbs(item, uniquecast) results.extend(data["result"][section]) chunk_start = (chunk * CHUNK_SIZE) response = {"result": {"limits": {"start": 0, "end": len(results), "total": len(results)}}} if section: response["result"][section] = results return response # Create a new cast list ignoring any cast member without a thumbnail. # Replace original cast list with the new cast list. def removecastwithoutthumbs(self, mediaitem, uniquecast=None): if "cast" in mediaitem: cast = [] for i in mediaitem["cast"]: if "thumbnail" in i: if uniquecast is not None: if i["thumbnail"] not in uniquecast: uniquecast[i["thumbnail"]] = True cast.append(i) else: cast.append(i) mediaitem["cast"] = cast # Return a list of all pictures (jpg/png/tbn etc.) from any "pictures" source def getPictures(self, addPreviews=False, addPictures=True): list = [] if addPreviews or addPictures: for path in self.getSources("pictures"): self.getPicturesForPath(path, list, addPreviews, addPictures) return list def getPicturesForPath(self, path, list, addPreviews, addPictures): data = self.getDirectoryList(path, use_cache=False) if "result" not in data or "files" not in data["result"]: return DIR_ADDED = False for file in data["result"]["files"]: if file["file"]: ftype = file["filetype"] fname = file["file"] if ftype == "directory": self.getPicturesForPath(fname, list, addPreviews, addPictures) elif ftype == "file" and os.path.splitext(fname)[1].lower() in self.config.picture_filetypes: if addPreviews: if not DIR_ADDED: DIR_ADDED = True list.append({"type": "directory", "label": path, "thumbnail": "picturefolder@%s" % path}) list.append({"type": "file", "label": fname, "thumbnail": "%s/transform?size=thumb" % fname}) if addPictures: list.append({"type": "file", "label": fname, "thumbnail": fname}) def parseSQLFilter(self, filter): if type(filter) is dict: return filter filter = filter.strip() if not filter: return [] if filter.lower().startswith("where "): filter = filter[6:] PATTERN = re.compile(r'''((?:[^ "']|"[^"]*"|'[^']*')+)''') stack = [] data = [] fields = [] condition = None group = False f = 0 for token in PATTERN.split(filter)[1::2]: #Unescape any escaped apostrophe token = token.replace("''","'") if token.startswith("("): group = False token = token[1:] elif token.endswith(")"): group = True token = token[:-1] if token in ["and", "or"]: if condition and condition != token: if group: if token in ["and", "or"]: if stack == []: stack.append({token: [{condition: data}]}) else: s = stack.pop() stack.append({token: [s]}) data = [] condition = token if not group else None continue fields.append(token) f += 1 if f == 3: if fields[0].startswith("t."): fields[0] = fields[0][2:] if fields[0] == "id": fields[0] = "textureid" if fields[0] == "lastusetime": fields[0] = "lastused" if (fields[2].startswith("'") and fields[2].endswith("'")) or \ (fields[2].startswith('"') and fields[2].endswith('"')): fields[2] = fields[2][1:-1] if fields[1] in ["=", "=="]: fields[1] = "is" elif fields[1] == "!=": fields[1] = "isnot" elif fields[1] == ">": fields[1] = "greaterthan" elif fields[1] == "<": fields[1] = "lessthan" elif fields[1] == ">=": fields[1] = "greaterthanequal" elif fields[1] == "<=": fields[1] = "lessthanequal" elif fields[1].lower() == "like": if re.match("^%.*%$", fields[2]): fields[1] = "contains" fields[2] = fields[2][1:-1] elif re.match("^%.*", fields[2]): fields[1] = "endswith" fields[2] = fields[2][1:] elif re.match("^.*%", fields[2]): fields[1] = "startswith" fields[2] = fields[2][:-1] else: fields[1] = "is" else: fields[1] = "is" pass if fields[1].endswith("thanequal"): data.append({"or": [{"field": fields[0], "operator": "is", "value": fields[2]}, {"field": fields[0], "operator": fields[1][:-5], "value": fields[2]}]}) else: data.append({"field": fields[0], "operator": fields[1], "value": fields[2]}) if group: if stack == []: stack.append({condition: data}) else: s = stack.pop() c1 = "or" if "or" in s else "and" c2 = "or" if "or" in s[c1][0] else "and" if condition: s[c1].append({condition: data}) else: s[c1].extend(data) stack.append(s) data = [] fields = [] f = 0 if stack: return stack[0] elif data: if condition: return {condition: data} else: return data[0] else: return data def parseSQLOrder(self, order): if type(order) is dict: return order order = order.strip() if not order: return [] if order.lower().startswith("order by "): order = order[9:] PATTERN = re.compile(r'''((?:[^ "']|"[^"]*"|'[^']*')+)''') data = [] fields = [] f = 0 for word in PATTERN.split(order)[1::2]: fields.append(word) f += 1 if f == 2: if fields[0].startswith("t."): fields[0] = fields[0][2:] if fields[0] == "id": fields[0] = "textureid" if fields[0] == "lastusetime": fields[0] = "lastused" if fields[1].lower().startswith("asc"): fields[1] = "ascending" else: fields[1] = "descending" data.append({"method": fields[0], "order": fields[1]}) break return data[0] if data else data def getTextures(self, filter=None, order=None, allfields=False): REQUEST = {"method": "Textures.GetTextures", "params": {"properties": ["cachedurl", "url"]}} if allfields: REQUEST["params"]["properties"].extend(["lasthashcheck", "imagehash", "sizes"]) if filter: param = self.parseSQLFilter(filter) if param: REQUEST["params"]["filter"] = param return self.sendJSON(REQUEST, "libTextures", checkResult=False) def delTexture(self, id): REQUEST = {"method": "Textures.RemoveTexture", "params": {"textureid": id}} return self.sendJSON(REQUEST, "libTextures", checkResult=False) # # Hold and print some pretty totals. # class MyTotals(object): def __init__(self, lastRunDateTime): self.LASTRUNDATETIME = lastRunDateTime self.TIMES = {} self.ETIMES = {} self.THREADS = {} self.THREADS_HIST = {} self.HISTORY = (time.time(), time.time(), 0) self.PCOUNT = self.PMIN = self.PAVG = self.PMAX = 0 self.MCOUNT = self.MMIN = self.MAVG = self.MMAX = 0 self.TOTALS = {} self.TOTALS["Skipped"] = {} self.TOTALS["Deleted"] = {} self.TOTALS["Duplicate"] = {} self.TOTALS["Error"] = {} self.TOTALS["Cached"] = {} self.TOTALS["Ignored"] = {} self.TOTALS["Undefined"] = {} def addSeasonAll(self): if not "Season-all" in self.TOTALS: self.TOTALS["Season-all"] = {} def addNotCached(self): if not "Not in Cache" in self.TOTALS: self.TOTALS["Not in Cache"] = {} def TimeStart(self, mediatype, item): if not mediatype in self.TIMES: self.TIMES[mediatype] = {} self.TIMES[mediatype][item] = (time.time(), 0) def TimeEnd(self, mediatype, item): self.TIMES[mediatype][item] = (self.TIMES[mediatype][item][0], time.time()) def TimeDuration(self, item): tElapsed = 0 for m in self.TIMES: for i in self.TIMES[m]: if i == item: tuple = self.TIMES[m][i] tElapsed += (tuple[1] - tuple[0]) return tElapsed def gotTimeDuration(self, item): for m in self.TIMES: for i in self.TIMES[m]: if i == item: return True return False def init(self, name=""): with lock: tname = threading.current_thread().name if name == "" else name self.THREADS[tname] = 0 self.THREADS_HIST[tname] = (0, 0) # Record start time for an image type. def start(self, mediatype, imgtype): with lock: tname = threading.current_thread().name ctime = time.time() self.THREADS[tname] = ctime if not mediatype in self.ETIMES: self.ETIMES[mediatype] = {} if not imgtype in self.ETIMES[mediatype]: self.ETIMES[mediatype][imgtype] = {} if not tname in self.ETIMES[mediatype][imgtype]: self.ETIMES[mediatype][imgtype][tname] = (ctime, 0) # Record current time for imgtype - this will allow stats to # determine cumulative time taken to download an image type. def finish(self, mediatype, imgtype): with lock: tname = threading.current_thread().name ctime = time.time() self.setPerformance(ctime - self.THREADS[tname]) self.THREADS_HIST[tname] = (self.THREADS[tname], ctime) self.THREADS[tname] = 0 self.ETIMES[mediatype][imgtype][tname] = (self.ETIMES[mediatype][imgtype][tname][0], ctime) def stop(self): self.init() # Increment counter for action/imgtype pairing def bump(self, action, imgtype): with lock: if not action in self.TOTALS: self.TOTALS[action] = {} if not imgtype in self.TOTALS[action]: self.TOTALS[action][imgtype] = 0 self.TOTALS[action][imgtype] += 1 # Calculate and store min/max/avg. def setPerformance(self, elapsed): with lock: (s, e, c) = self.HISTORY self.HISTORY = (s, time.time(), c+1) self.PCOUNT += 1 self.PAVG += elapsed if self.PMIN == 0 or elapsed < self.PMIN: self.PMIN = elapsed if elapsed > self.PMAX: self.PMAX = elapsed # Calculate average performance per second. def getPerformance(self, remaining): with lock: (s, e, c) = self.HISTORY tpersec = (c / (e - s)) if c != 0 else 1.0 eta = self.secondsToTime(remaining / tpersec, withMillis=False) return " (%05.2f downloads per second, ETA: %s)" % (tpersec, eta) def libraryStats(self, item="", multi=[], filter="", lastRun=False, query=""): if multi: item = "/".join(multi) # Determine the artwork types that have been accumulated items = {} for a in self.TOTALS: for c in self.TOTALS[a]: if c not in items: items[c] = None # Ensure some basic items are included in the summary if item.find("pvr.") != -1: if "thumbnail" not in items: items["thumbnail"] = None else: if "fanart" not in items: items["fanart"] = None if item.find("movies") != -1: if "poster" not in items: items["poster"] = None if item.find("tvshows") != -1: if "thumb" not in items: items["thumb"] = None if item.find("artists") != -1 or \ item.find("albums") != -1 or \ item.find("songs") != -1: if "thumbnail" not in items: items["thumbnail"] = None DOWNLOAD_LABEL = "Download Time" sortedItems = sorted(items.items()) sortedItems.append(("TOTAL", None)) items["TOTAL"] = 0 sortedTOTALS = sorted(self.TOTALS.items()) sortedTOTALS.append(("TOTAL", {})) self.TOTALS["TOTAL"] = {} if len(self.THREADS_HIST) != 0: sortedTOTALS.append((DOWNLOAD_LABEL, {})) self.TOTALS[DOWNLOAD_LABEL] = {"TOTAL": 0} # Transfer elapsed times for each image type to our matrix of values # Times are held by mediatype, so accumulate for each mediatype # Total Download Time is sum of elapsed time for each mediatype self.TOTALS[DOWNLOAD_LABEL]["TOTAL"] = 0 for mtype in self.ETIMES: tmin = tmax = mmin = mmax = 0.0 for itype in self.ETIMES[mtype]: itmin = itmax = mtmin = mtmax = 0.0 for tname in self.ETIMES[mtype][itype]: tuple = self.ETIMES[mtype][itype][tname] if tname.startswith("Main"): mtmin = tuple[0] mtmax = tuple[1] else: if tuple[0] < itmin or itmin == 0.0: itmin = tuple[0] if tuple[1] > itmax: itmax = tuple[1] if itype not in self.TOTALS[DOWNLOAD_LABEL]: self.TOTALS[DOWNLOAD_LABEL][itype] = 0 self.TOTALS[DOWNLOAD_LABEL][itype] = (itmax - itmin) + (mtmax - mtmin) if itmin < tmin or tmin == 0.0: tmin = itmin if itmax > tmax: tmax = itmax if mtmin < mmin or mmin == 0.0: mmin = mtmin if mtmax > mmax: mmax = mtmax self.TOTALS[DOWNLOAD_LABEL]["TOTAL"] += (tmax - tmin) + (mmax - mmin) line0 = "Cache pre-load activity summary for \"%s\"" % item if filter != "": line0 = "%s, filtered by \"%s\"" % (line0, filter) if lastRun and self.LASTRUNDATETIME: line0 = "%s, added since %s" % (line0, self.LASTRUNDATETIME) line0 = "%s:" % line0 line1 = "%-14s" % " " line2 = "-" * 14 for i in sortedItems: i = i[1] if i[1] else i[0] width = 12 if len(i) < 12 else len(i)+1 line1 = "%s| %s" % (line1, i.center(width)) line2 = "%s+-%s" % (line2, "-" * width) print("") print(line0) print("") print(line1) print(line2) for a in sortedTOTALS: a = a[0] if a != DOWNLOAD_LABEL: self.TOTALS[a]["TOTAL"] = 0 if a == "TOTAL": print(line2.replace("-","=").replace("+","=")) line = "%-13s " % a for i in sortedItems: i = i[0] if a == "TOTAL": value = "%d" % items[i] if items[i] is not None else "-" elif a == DOWNLOAD_LABEL: if i in self.TOTALS[a] and self.TOTALS[a][i] != 0: value = self.secondsToTime(self.TOTALS[a][i]) else: value = "-" elif i in self.TOTALS[a]: ivalue = self.TOTALS[a][i] value = "%d" % ivalue if items[i] is None: items[i] = 0 items[i] += ivalue self.TOTALS[a]["TOTAL"] += ivalue else: value = "-" width = 12 if len(i) < 12 else len(i)+1 line = "%s| %s" % (line, value.center(width)) print(line) print("") self.libraryStatsSummary() def libraryStatsSummary(self): # Failed to load anything so don't display time stats that we don't have if not self.gotTimeDuration("Load"): return if len(self.THREADS_HIST) != 0: tcount = 0 pcount = 1 if self.PCOUNT == 0 else self.PCOUNT mavg = 1 if self.MAVG == 0.0 else self.MAVG for tname in self.THREADS_HIST: if not tname.startswith("Main"): tcount += 1 print(" Threads Used: %d" % tcount) print(" Min/Avg/Max: %05.2f / %05.2f / %05.2f downloads per second" % (self.MMIN, self.MCOUNT/mavg, self.MMAX)) print(" Min/Avg/Max: %05.2f / %05.2f / %05.2f seconds per download" % (self.PMIN, self.PAVG/pcount, self.PMAX)) print("") print(" Loading: %s" % self.secondsToTime(self.TimeDuration("Load"))) print(" Parsing: %s" % self.secondsToTime(self.TimeDuration("Parse"))) if self.gotTimeDuration("Compare"): print(" Comparing: %s" % self.secondsToTime(self.TimeDuration("Compare"))) if self.gotTimeDuration("Rescan"): print(" Rescanning: %s" % self.secondsToTime(self.TimeDuration("Rescan"))) if self.gotTimeDuration("PreDelete"): print(" Pre-Deleting: %s" % self.secondsToTime(self.TimeDuration("PreDelete"))) if len(self.THREADS_HIST) != 0: print(" Downloading: %s" % self.secondsToTime(self.TimeDuration("Download"))) print(" TOTAL RUNTIME: %s" % self.secondsToTime(self.TimeDuration("Total"))) def secondsToTime(self, seconds, withMillis=True): ms = int(100 * (seconds - int(seconds))) m, s = divmod(seconds, 60) h, m = divmod(m, 60) d, h = divmod(h, 24) if d == 0: t = "%02d:%02d:%02d" % (h, m, s) else: t = "%dd %02d:%02d:%02d" % (d, h, m, s) if withMillis: t = "%s.%02d" % (t, ms) return t # # Simple container for media items under consideration for processing # # status is 0 (not to be loaded) or 1 (to be loaded). # # missingOK is used to specify that if an image cannot be loaded, # don't complain (ie. speculative loading, eg. season-all.tbn) # class MyMediaItem(object): # 0=Ignore/Skipped; 1=Missing, to be cached; 2=Stale, to be cached; 3=Queued for downloading STATUS_UNKNOWN = 0 STATUS_IGNORE = 1 STATUS_MISSING = 2 STATUS_STALE = 3 STATUS_QUEUED = 4 def __init__(self, mediaType, imageType, name, season, episode, filename, dbid, cachedurl, libraryid, missingOK): self.status = MyMediaItem.STATUS_UNKNOWN self.mtype = mediaType self.itype = imageType self.name = name self.season = season self.episode = episode self.filename = filename self.decoded_filename = MyUtility.normalise(self.filename, strip=True) if self.filename else self.filename self.dbid = dbid self.cachedurl = cachedurl self.libraryid = libraryid self.missingOK = missingOK def __str__(self): season = "\"%s\"" % self.season if self.season else self.season episode = "\"%s\"" % self.episode if self.episode else self.episode cachedurl = "\"%s\"" % self.cachedurl if self.cachedurl else self.cachedurl return "{%d, \"%s\", \"%s\", \"%s\", %s, %s, \"%s\", %d, %s, %s, %s}" % \ (self.status, self.mtype, self.itype, self.name, season, episode, \ self.decoded_filename, self.dbid, cachedurl, \ self.libraryid, self.missingOK) def getTypeSingular(self): return self.mtype[:-1] def getFullName(self): if self.episode: if self.mtype == "tvshows": return "%s, %s Episode %s" % (self.name, self.season, self.episode) elif self.mtype == "songs": return "%s from: %s by: %s" % (self.name, self.episode, " & ".join(self.season)) else: return "%s, %s: %s" % (self.name, self.season, self.episode) elif self.season: if self.itype == "cast.thumb": return "%s in %s" % (self.name, self.season) elif self.mtype == "tvshows": return "%s, %s" % (self.name, self.season) elif self.mtype.startswith("pvr."): return "%s (%s)" % (self.season, self.name) elif self.mtype in ["albums", "songs"]: return "%s by: %s" % (self.name, " & ".join(self.season)) else: return "%s, %s" % (self.name, self.season) else: return "%s" % self.name # # Simple container for watched items. # class MyWatchedItem(object): def __init__(self, mediaType, name, episode_year, playcount, lastplayed, resume): self.mtype = mediaType self.name = name self.episode_year = episode_year self.playcount = int(playcount) self.lastplayed = lastplayed self.resume = resume self.libraryid = 0 # 0 = Valid (write to Media Library) # 1 = Unchanged - same as Media Library (don't update Media Library) # 2 = Out of Date - Media Library has been updated since backup list created self.state = 0 if self.episode_year is None: self.episode_year = "" def __str__(self): return "['%s', %d, '%s', '%s', %d, '%s, %s']" % \ (self.mtype, self.libraryid, self.name, self.episode_year, self.playcount, self.lastplayed, self.resume) def getList(self): return [self.mtype, self.libraryid, self.name, self.episode_year, self.playcount, self.lastplayed, self.resume] def match(self, mediatype, name, episode_year): if mediatype != self.mtype: return False xepisode_year = episode_year if xepisode_year is None: xepisode_year = "" return (self.name == name and self.episode_year == xepisode_year) def refresh(self, HAS_RESUME, playcount, lastplayed, resume): # Update this object to reflect most recent (latest) values if playcount > self.playcount: self.playcount = playcount if lastplayed > self.lastplayed: self.lastplayed = lastplayed if HAS_RESUME: if resume["position"] > self.resume["position"]: self.resume["position"] = resume["position"] def setState(self, HAS_RESUME, playcount, lastplayed, resume): # Assume no change is required self.state = 1 if self.playcount == playcount and self.lastplayed == lastplayed: if not HAS_RESUME: return if self.resume["position"] == resume["position"]: return # Something has changed, apply object values to library self.state = 0 return # Helper class... class MyUtility(object): isPython3 = (sys.version_info >= (3, 0)) isPython3_1 = (sys.version_info >= (3, 1)) EPOCH = datetime.datetime.utcfromtimestamp(0) DCData = {} DCStats = {} DCStatsAccumulated = {} #http://kodi.wiki/view/Advancedsettings.xml#moviestacking # #(.*?)([ _.-]*(?:cd|dvd|p(?:ar)?t|dis[ck])[ _.-]*[0-9]+)(.*?)(\.[^.]+)$ # #(.*?)([ _.-]*(?:cd|dvd|p(?:ar)?t|dis[ck])[ _.-]*[a-d])(.*?)(\.[^.]+)$ RE_STACKING_1_9 = re.compile("(.*?)([ _.-]*(?:cd|dvd|p(?:ar)?t|dis[ck])[ _.-]*[0-9]+)(.*?)(\.[^.]+)$", flags=re.IGNORECASE) RE_STACKING_A_D = re.compile("(.*?)([ _.-]*(?:cd|dvd|p(?:ar)?t|dis[ck])[ _.-]*[a-d])(.*?)(\.[^.]+)$", flags=re.IGNORECASE) RE_NOT_DIGITS = re.compile("[^0123456789]") # Convert quoted filename into consistent UTF-8 # representation for both Python2 and Python3 @staticmethod def normalise(value, strip=False): if not value: return value v = urllib2.unquote(value) if strip: s = 8 if v.startswith("image://") else None if s: e = -1 if v[-1:] == "/" else None v = v[s:e] if not MyUtility.isPython3: try: v = bytes(v.encode("iso-8859-1")).decode("utf-8") except UnicodeDecodeError: pass except UnicodeEncodeError: pass return v # Quote unquoted filename @staticmethod def denormalise(value, prefix=True): v = value if not MyUtility.isPython3: try: v = bytes(v.encode("utf-8")) except UnicodeDecodeError: pass v = urllib2.quote(v, "") if prefix: v = "image://%s/" % v return MyUtility.toUnicode(v) @staticmethod def toUnicode(data): if MyUtility.isPython3: return data if isinstance(data, basestring): if not isinstance(data, unicode): try: data = unicode(data, encoding="utf-8", errors="ignore") except UnicodeDecodeError: pass return data # Cross-platform basename # os.path.basename() doesn't work when # processing a Windows filename on Linux @staticmethod def basename(filename): # If X:\ then assume a Windows filename if filename[1:].startswith(':\\'): return filename.split('\\')[-1] # Otherwise assume Linux or network (smb:// etc.) based filename return filename.split('/')[-1] # Join an unquoted filename to a quoted path, # returning a quoted result. # # Running urllib2.quote() on a path that contains # foreign characters would often fail with a Unicode error # so avoid the quote() call entirely (except on the filename # which should be safe (as this is only called from getSeasonAll() # so the only filenames are season-all-poster etc.). # @staticmethod def joinQuotedPath(qpath, filename): # Remove filename, leaving just directory. # Could use dirname() here but the path is quoted # and we need to know which slash is being used so # that it can be re-appended for qslash in ["%2f", "%2F", "%5c", "%5C"]: pos = qpath.rfind(qslash) if pos != -1: directory = "%s%s" % (qpath[:pos], qslash) break else: return None fname = urllib2.quote(MyUtility.basename(filename), "") return "%s%s/" % (directory, fname) # # Some JSON paths may have incorrect path separators. # Use this function to attempt to correct those path separators. # # Shares ("smb://", "nfs://" etc.) will always use forward slashes. # # Non-shares will use a slash appropriate to the OS to which the path # corresponds so attempt to find the FIRST slash (forward or back) and # then use that as the path separator, replacing any of the opposite # kind. The reason being that path mangling addons are likely to mangle # only the last slashes but not the first. # # If only one type of slash found (or neither slash found), do nothing. # # See: http://forum.kodi.tv/showthread.php?tid=153502&pid=1477147#pid1477147 # @staticmethod def fixSlashes(filename): # Share (eg. "smb://", "nfs://" etc.) if re.search("^.*://.*", filename): return filename.replace("\\", "/") bslash = filename.find("\\") fslash = filename.find("/") if bslash == -1 or fslash == -1: return filename elif bslash < fslash: return filename.replace("/", "\\") else: #fslash < bslash: return filename.replace("\\", "/") # Convert a path or filename from Windows or Linux format, to the "host" OS format @staticmethod def PathToHostOS(filename): # Share (eg. "smb://", "nfs://" etc.) if re.search("^.*://.*", filename): return filename.replace("\\", "/") if os.sep == "/": return filename.replace("\\", "/") else: return filename.replace("/", "\\") @staticmethod def Top250MovieList(): gLogger.progress("Retrieving Top250 movie rankings...") try: import xml.etree.ElementTree as ET URL = "http://top250.info/charts" gLogger.log("Top250: Retrieving Top250 Movies from: [%s]" % URL) html = urllib2.urlopen(URL) if MyUtility.isPython3: data = html.read().decode('utf-8') else: data = html.read() html.close() gLogger.log("Top250: Read %d bytes of data" % len(data)) tstamp = data.find("") if tstamp != -1: tstamp += 30 gLogger.log("Top250: Data last updated [%s]" % data[tstamp:tstamp+18]) # Find end of first table, as movie list is in the second table spos = data.find("</table>") epos = data[spos+8:].find("</table>") if spos != -1 else -1 if spos != -1 and epos != -1: data = data[spos+8:spos+8+epos+8] gLogger.log("Top250: Table data found, %d bytes" % len(data)) # Clean up garbage encodings newdata = "" for c in data: if c != "&": if ord(c) <= 127: newdata += c table = ET.fromstring(newdata) RE_IMDB = re.compile("/movie/\?([0-9]*)") movies = {} for row in table: if row.attrib["class"] == "row_header": continue movie = {} title = row[2][0][0].text anchor = row[2][0].attrib["href"] s = RE_IMDB.search(anchor) if s: movie["link"] = "tt%s" % s.group(1) movie["title"] = title movie["rank"] = len(movies)+1 try: movie["rating"] = float(row[3].text) except: pass try: if gConfig.JSON_VOTES_HAVE_NO_GROUPING: movie["votes"] = u"%s" % int(MyUtility.getDigits(row[4].text)) else: movie["votes"] = u"%s" % format(int(MyUtility.getDigits(row[4].text)), ",d").replace(",", gConfig.IMDB_GROUPING) except: pass movies[movie["link"]] = movie gLogger.log("Top250: loaded %d movies" % len(movies)) return movies else: gLogger.log("Top250: ERROR: didn't find movie data, skipping Top250 movies") return None except Exception as e: gLogger.log("Top250: ERROR: failed to retrieve Top250 movie data: [%s]" % str(e)) raise return None @staticmethod def getIMDBInfo(mediatype, apikey, imdbnumber=None, title=None, year=None, season=None, episode=None, plotFull=False, plotOutline=False, qtimeout=15.0): try: omdb_url = "http://www.omdbapi.com/" base_url = "%s?apikey=%s" % (omdb_url, apikey) redacted_url = "%s?apikey=%s" % (omdb_url, "<yourkey>") isMovie = isTVShow = isSeason = isEpisode = False if mediatype == "movie": query_url = "i=%s&type=movie" % imdbnumber reference = imdbnumber isMovie = True elif mediatype in ["tvshow", "episode"]: reference = "tvshow: %s (%d)" % (title, year) if imdbnumber is not None: query_url = "i=%s" % imdbnumber else: query_url = "t=%s&y=%d" % (MyUtility.denormalise(title, False), year) isTVShow = True if season: query_url = "%s&season=%d" % (query_url, season) reference = "season: %s (%d) S%02d" % (title, year, season) isSeason = True if episode: query_url = "%s&episode=%d" % (query_url, episode) reference = "episode: %s (%d) S%02dE%02d" % (title, year, season, episode) isEpisode = True query_url = "%s&type=%s" % (query_url, "episode" if isEpisode else "series") else: return {} data_short = data_full = data = {} # For movie, get both short plot, and optionally the full plot. Use fields from short query. if isMovie: f = urllib2.urlopen("%s&%s&plot=short" % (base_url, query_url), timeout=qtimeout) data_short = json.loads(f.read().decode("utf-8")) f.close() # For TV shows and Episodes, we only need the full plot and fields. if not isMovie or plotFull: f = urllib2.urlopen("%s&%s&plot=full" % (base_url, query_url), timeout=qtimeout) data_full = json.loads(f.read().decode("utf-8")) f.close() if data_short != {}: data = data_short data["Plotoutline"] = data_short.get("Plot", None) data["Plot"] = None if data_full != {}: if data == {}: data = data_full data["Plot"] = data_full.get("Plot", None) if "Response" not in data or data["Response"] == "False": gLogger.log("Failed OMDb API Query [%s&%s] => [%s]" % (redacted_url, query_url, data)) if isTVShow and not isEpisode: gLogger.log("Try OMDb API Query [%s&s=%s&type=series] to see possible available titles (hint: year of tvshow is %d)" %(redacted_url, title, year)) return {} # Convert omdbapi.com fields to Kodi fields - mostly just a case # of converting to lowercase, and removing "imdb" prefix newdata = {} for key in data: if data[key] is None or data[key] == "N/A": continue newkey = key.replace("imdb", "").lower() try: # Convert rating from str to float if newkey == "rating": newdata[newkey] = float(data[key]) # Munge plot/plotoutline together as required elif newkey == "plot": if plotFull: newdata["plot"] = data.get(key, None) elif newkey == "plotoutline": if plotOutline: newdata["plotoutline"] = data.get(key, None) # Convert genre to a list elif newkey in ["genre", "country", "director", "writer"]: newdata[newkey] = [g.strip() for g in data[key].split(",")] # Year to an int elif newkey == "year": if not isMovie: continue newdata[newkey] = int(data[key]) # Runtime from "2 h", "36 min", "2 h 22 min" or "N/A" to seconds elif newkey == "runtime": t = data[key] h = re.search("([0-9]+) h", t) m = re.search("([0-9]+) min", t) r = 0 r += (int(h.group(1))*3600) if h else 0 r += (int(m.group(1))*60) if m else 0 if r > 0: newdata[newkey] = r elif newkey == "votes": if gConfig.JSON_VOTES_HAVE_NO_GROUPING: newdata[newkey] = "%s" % int(MyUtility.getDigits(data[key])) else: newdata[newkey] = format(int(MyUtility.getDigits(data[key])), ",d").replace(",", gConfig.IMDB_GROUPING) elif newkey == "rated": newkey = "mpaa" if data[key] == None or data[key] in ["Not Rated", "Unrated"]: newdata[newkey] = "Not Rated" else: newdata[newkey] = "Rated %s" % data[key] elif newkey == "released": newkey = "premiered" newdata[newkey] = datetime.datetime.strptime(data[key].replace(" ","-"), '%d-%b-%Y').strftime("%Y-%m-%d") else: newdata[newkey] = data[key] if newkey in newdata: newdata[newkey] = MyUtility.toUnicode(newdata[newkey]) except Exception as e: gLogger.log("Exception during IMDb processing: reference [%s], key [%s]. msg [%s]" % (reference, key, str(e))) gLogger.log("OMDb API Query [%s&%s]" % (redacted_url, query_url)) return newdata except Exception as e: gLogger.log("Exception during IMDb processing: reference [%s], timeout [%s], msg [%s]" % (reference, qtimeout, str(e))) gLogger.log("OMDb API Query [%s&%s]" % (redacted_url, query_url)) return {} @staticmethod def nonestr(s): if s is None: return "" return str(s) @staticmethod def is_cache_item_stale(config, jcomms, mediaitem): if config.cache_refresh_date is None: return False if mediaitem.decoded_filename.startswith("http://"): return False # Only check file details for the following media types if mediaitem.mtype in ["movies", "tags", "sets", "tvshows", "seasons", "episodes", "albums", "artists", "songs"]: file = jcomms.getFileDetails(mediaitem.decoded_filename, properties=["file", "lastmodified"]) if file and "lastmodified_timestamp" in file: return (file["lastmodified_timestamp"] >= config.cache_refresh_date) return False @staticmethod def invalidateDirectoryCache(mediatype): with lock: # Transfer current stats to accumulated for p in MyUtility.DCData: for c in MyUtility.DCStats[p]: if c not in MyUtility.DCStatsAccumulated: MyUtility.DCStatsAccumulated[c] = 0 MyUtility.DCStatsAccumulated[c] += MyUtility.DCStats[p][c] MyUtility.logDirectoryCacheStats(mediatype, totals=False) del MyUtility.DCData del MyUtility.DCStats MyUtility.DCData = {} MyUtility.DCStats = {} @staticmethod def setDirectoryCacheItem(data, properties, path): props = ",".join(sorted(properties)) fs_bs = "\\" if path.find("\\") != -1 else "/" if path[-1:] != fs_bs: path += fs_bs with lock: if props not in MyUtility.DCData: MyUtility.DCData[props] = {} if props not in MyUtility.DCStats: MyUtility.DCStats[props] = {"miss": 0, "store": 0, "hit": 0, "evicted": 0} count = MyUtility.DCData[props].get(path, {"count": 0})["count"] MyUtility.DCData[props][path] = {"time": time.time(), "count": count+1, "data": data} if gConfig.LOGDCACHE: hits = MyUtility.DCData[props][path]["count"] size = len(MyUtility.DCData[props]) gLogger.log("Directory Cache %4s: %s (%s) [hit #1. %d items in cache]" % ("STOR", props, path, size)) # If we've just added a new item, we may need to now trim the cache if count == 0: MyUtility.DCStats[props]["store"] += 1 MyUtility.trimDirectoryCache(props) @staticmethod def getDirectoryCacheItem(properties, path): props = ",".join(sorted(properties)) fs_bs = "\\" if path.find("\\") != -1 else "/" if path[-1:] != fs_bs: path += fs_bs with lock: if props not in MyUtility.DCData: MyUtility.DCData[props] = {} if props not in MyUtility.DCStats: MyUtility.DCStats[props] = {"miss": 1, "store": 0, "hit": 0, "evicted": 0} result = None elif path not in MyUtility.DCData[props]: MyUtility.DCStats[props]["miss"] += 1 result = None else: MyUtility.DCStats[props]["hit"] += 1 c = MyUtility.DCData[props][path] c["time"] = time.time() c["count"] += 1 result = c["data"] if gConfig.LOGDCACHE: hits = MyUtility.DCData[props][path]["count"] if result else 1 size = len(MyUtility.DCData[props]) gLogger.log("Directory Cache %4s: %s (%s) [hit #%d, %d items in cache]" % (("HIT " if result else "MISS"), props, path, hits, size)) return result @staticmethod def trimDirectoryCache(properties): if properties not in MyUtility.DCData: return if len(MyUtility.DCData[properties]) <= gConfig.DCACHE_SIZE: return cp = MyUtility.DCData[properties] now = time.time() cexpiry = now - gConfig.DCACHE_AGELIMIT delKeys = [] # Identify any cache items that can be expired due to old age for citem in cp: if cp[citem]["time"] < cexpiry: delKeys.append(citem) for ditem in delKeys: if gConfig.LOGDCACHE: gLogger.log("Directory Cache TRIM: %s (%s) [%d hits, %d items] (Age: %5.2f seconds)" % (properties, ditem, MyUtility.DCData[properties][ditem]["count"], len(MyUtility.DCData[properties]), (now - cp[ditem]["time"]))) del cp[ditem] if properties in MyUtility.DCStats: MyUtility.DCStats[properties]["evicted"] += len(delKeys) # If we still need to trim the cache # to its maximum size, get rid of the oldest item(s) while len(cp) > gConfig.DCACHE_SIZE: oldestTime = now oldestItem = None for citem in cp: if cp[citem]["time"] < oldestTime: oldestTime = cp[citem]["time"] oldestItem = citem if oldestItem: if properties in MyUtility.DCStats: MyUtility.DCStats[properties]["evicted"] += 1 if gConfig.LOGDCACHE: gLogger.log("Directory Cache TRIM: %s (%s) [%d hits, %d items] (Size)" % (properties, oldestItem, cp[oldestItem]["count"], len(cp))) del cp[oldestItem] @staticmethod def logDirectoryCacheStats(mediatype=None, totals=False): if gLogger.LOGGING: if totals and MyUtility.DCStatsAccumulated: gLogger.log("Directory Cache Config: Maximum Size %d, Age Limit %d seconds" % (gConfig.DCACHE_SIZE, gConfig.DCACHE_AGELIMIT)) stats = MyUtility.DCStatsAccumulated gLogger.log("Directory Cache Totals: Misses %d, Stores %d, Hits %d, Evicted %d" % (stats["miss"], stats["store"], stats["hit"], stats["evicted"])) else: for props in MyUtility.DCStats: stats = MyUtility.DCStats[props] gLogger.log("Directory Cache PERF: Misses %d, Stores %d, Hits %d, Evicted %d, mediatype [%s] for properties: %s" % (stats["miss"], stats["store"], stats["hit"], stats["evicted"], mediatype, props.split(","))) @staticmethod def SinceEpoch(dt): return int((dt - MyUtility.EPOCH).total_seconds()) @staticmethod def getVersion(strVersion): fields = strVersion.split(".") return int("%03d%03d%03d" % (int(fields[0]), int(fields[1]), int(fields[2]))) @staticmethod def removeDiscPart(filename): match = MyUtility.RE_STACKING_1_9.match(filename) if not match: match = MyUtility.RE_STACKING_A_D.match(filename) if match and len(match.groups()) >= 3: p1 = match.string[:match.end(1)] p2 = match.string[match.start(3):] p1 = p1[:-1] if p1[-1] in [" ", "(", "[", ".", "-", "_"] else p1 p2 = p2[1:] if p2[0] in [")", "]"] else p2 return "%s%s" % (p1, p2) return filename @staticmethod def unstackFiles(files, addcombinedfile=False): if files.startswith("stack://"): unstack = files[8:].split(" , ") if addcombinedfile: unstack.insert(0, MyUtility.removeDiscPart(unstack[0])) return unstack else: return [files] @staticmethod def getDigits(text): return MyUtility.RE_NOT_DIGITS.sub("", text) # # Load data using JSON-RPC. In the case of TV shows, also load seasons # and Episodes into a single data structure. # # Sets doesn't support filters, so filter this list after retrieval. # def jsonQuery(action, mediatype, filter="", force=False, extraFields=False, rescan=False, \ decode=False, ensure_ascii=True, nodownload=False, lastRun=False, \ labels=None, query="", filename=None, wlBackup=True, drop_items=None): if mediatype not in ["addons", "agenres", "vgenres", "albums", "artists", "songs", "musicvideos", "movies", "sets", "tags", "tvshows", "pvr.tv", "pvr.radio"]: gLogger.err("ERROR: %s is not a valid media class" % mediatype, newLine=True) sys.exit(2) # Only songs, movies and tvshows (and sub-types) valid for missing... if action == "missing" and mediatype not in ["songs", "movies", "tvshows", "seasons", "episodes"]: gLogger.err("ERROR: media class [%s] is not currently supported by missing" % mediatype, newLine=True) sys.exit(2) # Only movies and tvshows for "watched"... if action == "watched" and mediatype not in ["movies", "tvshows"]: gLogger.err("ERROR: media class [%s] is not currently supported by watched" % mediatype, newLine=True) sys.exit(2) # Only movies and tvshows for "imdb"... if action == "imdb" and mediatype not in ["movies", "tvshows"]: gLogger.err("ERROR: media class [%s] is not currently supported by IMDb" % mediatype, newLine=True) return TOTALS.TimeStart(mediatype, "Total") jcomms = MyJSONComms(gConfig, gLogger) database = MyDB(gConfig, gLogger) if mediatype == "tvshows": TOTALS.addSeasonAll() gLogger.progress("Loading TV shows...") else: gLogger.progress("Loading %s..." % mediatype.capitalize()) TOTALS.TimeStart(mediatype, "Load") if action == "query": secondaryFields = parseQuery(query)[0] else: secondaryFields = None # Keep a hash of unique cast thumbnails to minimise number of cast member # details that are loaded from the client - the vast majority will be # duplicates that can be discarded. UCAST = {} if mediatype in ["pvr.tv", "pvr.radio"] and not gConfig.HAS_PVR: (section_name, title_name, id_name, data) = ("", "", "", []) elif mediatype == "vgenres": _data = [] for subtype in ["movie", "tvshow", "musicvideo"]: (section_name, title_name, id_name, data) = jcomms.getData(action, mediatype, filter, extraFields, lastRun=lastRun, secondaryFields=secondaryFields, subType=subtype) filter_name = gConfig.FILTER_FIELD if gConfig.FILTER_FIELD else title_name if data and "result" in data and section_name in data["result"]: if filter != "": filteredData = [] for d in data["result"][section_name]: if re.search(filter, d[filter_name], re.IGNORECASE): filteredData.append(d) data["result"][section_name] = filteredData if len(data["result"][section_name]) > 0: _data.append({"type": subtype, section_name: data["result"][section_name]}) title_name = "type" section_name = mediatype data["result"] = {section_name: _data} else: (section_name, title_name, id_name, data) = jcomms.getData(action, mediatype, filter, extraFields, lastRun=lastRun, secondaryFields=secondaryFields, uniquecast=UCAST) if data and "result" in data and section_name in data["result"]: data = data["result"][section_name] else: data = [] # Manually filter these mediatypes as JSON doesn't support filtering if data and filter and mediatype in ["addons", "agenres", "sets", "pvr.tv", "pvr.radio"]: filter_name = gConfig.FILTER_FIELD if gConfig.FILTER_FIELD else title_name gLogger.log("Filtering %s on %s = %s" % (mediatype, filter_name, filter)) filteredData = [] for d in data: if re.search(filter, d[filter_name], re.IGNORECASE): filteredData.append(d) data = filteredData # Add movie file members to sets when dumping if action == "dump" and data: if mediatype == "sets" and gConfig.ADD_SET_MEMBERS: gLogger.progress("Loading Sets members...") (s, t, i, fdata) = jcomms.getData(action, "sets-members", filter, extraFields, lastRun=lastRun, secondaryFields=None) if fdata and "result" in fdata and s in fdata["result"]: set_files = {} for movie in fdata["result"][s]: set_name = movie["set"] if set_name: del movie["set"] if "label" in movie: del movie["label"] if set_name not in set_files: set_files[set_name] = [] set_files[set_name].append(movie) for set in data: set["tc.members"] = set_files.get(set["title"], []) if mediatype == "albums" and gConfig.ADD_SONG_MEMBERS: gLogger.progress("Loading Song members...") (s, t, i, fdata) = jcomms.getData(action, "song-members", filter, extraFields, lastRun=lastRun, secondaryFields=None) if fdata and "result" in fdata and s in fdata["result"]: album_files = {} for album in fdata["result"][s]: album_name = album["album"] if album_name: del album["album"] if "label" in album: del album["label"] if album_name not in album_files: album_files[album_name] = [] album_files[album_name].append(album) for album in data: album["tc.members"] = album_files.get(album["title"], []) # Combine PVR channelgroups with PVR channels to create a hierarchical structure that can be parsed if mediatype in ["pvr.tv", "pvr.radio"]: pvrdata = [] for cg in data: (s1, t1, i1, data1) = jcomms.getData(action, "%s.channel" % mediatype, filter, extraFields, channelgroupid=cg["channelgroupid"], lastRun=lastRun, secondaryFields=secondaryFields) if "result" in data1: channels = [] for channel in data1["result"].get(s1, []): if "label" in channel: del channel["label"] channels.append(channel) pvrdata.append({"label": cg["label"], "channeltype": cg["channeltype"], "channelgroupid": cg["channelgroupid"], "channels": channels}) data = pvrdata if mediatype == "tvshows" and gConfig.QUERY_SEASONS: for tvshow in data: title = tvshow["title"] gLogger.progress("Loading TV show: %s..." % title) (s2, t2, i2, data2) = jcomms.getData(action, "seasons", filter, extraFields, tvshow=tvshow, lastRun=lastRun, uniquecast=UCAST) if not "result" in data2: return limits = data2["result"]["limits"] if limits["total"] == 0: continue tvshow[s2] = data2["result"][s2] for season in tvshow[s2]: seasonid = season["season"] if seasonid < 0: gLogger.err("WARNING: TV show [%s] has invalid season (%d) - ignored" % (title, seasonid), newLine=True) continue gLogger.progress("Loading TV show: %s, season %d..." % (title, seasonid)) if gConfig.QUERY_EPISODES: (s3, t3, i3, data3) = jcomms.getData(action, "episodes", filter, extraFields, tvshow=tvshow, tvseason=season, lastRun=lastRun, secondaryFields=secondaryFields, uniquecast=UCAST) if not "result" in data3: return limits = data3["result"]["limits"] if limits["total"] == 0: continue season[s3] = data3["result"][s3] del UCAST if lastRun and mediatype in ["movies", "tvshows"]: # Create a new list containing only tvshows with episodes... if mediatype == "tvshows": newData = [] for tvshow in data: newtvshow = {} epCount = 0 for season in tvshow.get("seasons", {}): if season.get("episodes", None): if newtvshow == {}: newtvshow = tvshow del newtvshow["seasons"] newtvshow["seasons"] = [] newtvshow["seasons"].append(season) epCount += len(season.get("episodes", {})) if newtvshow != {}: newData.append(newtvshow) gLogger.out("Recently added TV show: %s (%d episode%s)" % (tvshow.get("title"), epCount, "s"[epCount==1:]), newLine=True) data = newData else: for item in data: gLogger.out("Recently added movie: %s" % item.get("title", item.get("artist", item.get("name", None))), newLine=True) if len(data) != 0: gLogger.out("", newLine=True) TOTALS.TimeEnd(mediatype, "Load") if data != []: if action == "cache": cacheImages(mediatype, jcomms, database, data, title_name, id_name, force, nodownload, drop_items) elif action == "qa": qaData(mediatype, jcomms, database, data, title_name, id_name, rescan) elif action == "dump": jcomms.dumpJSON(data, decode, ensure_ascii) elif action == "missing": fileList = jcomms.getAllFilesForSource(mediatype, labels, gConfig.MISSING_IGNORE_PATTERNS, True) missingFiles(mediatype, data, fileList, title_name, id_name) elif action == "query": queryLibrary(mediatype, query, data, title_name, id_name) elif action == "watched" and wlBackup: watchedBackup(mediatype, filename, data, title_name, id_name) elif action == "watched" and not wlBackup: watchedRestore(mediatype, jcomms, filename, data, title_name, id_name) elif action == "duplicates": duplicatesList(mediatype, jcomms, data) elif action == "imdb": updateIMDb(mediatype, jcomms, data) else: raise ValueError("Unknown action [%s]" % action) # Free memory used to cache any GetDirectory() information MyUtility.invalidateDirectoryCache(mediatype) gLogger.progress("") TOTALS.TimeEnd(mediatype, "Total") # # Parse the supplied JSON data, turning it into a list of artwork urls # (mediaitems) that should be matched against the database (cached files) # to determine which should be skipped (those in the cache, unless # force update is true). # # Those that are not skipped will be added to a queueu for processing by # 1..n threads. Errors will be added to an error queue by the threads, and # subsueqently displayed to the user at the end. # def cacheImages(mediatype, jcomms, database, data, title_name, id_name, force, nodownload, drop_items): mediaitems = [] imagecache = {} imagecache[""] = 0 # Ensure an empty image is already in the imagecache, with a zero reference count TOTALS.TimeStart(mediatype, "Parse") parseURLData(jcomms, mediatype, mediaitems, imagecache, data, title_name, id_name) TOTALS.TimeEnd(mediatype, "Parse") # Don't need this data anymore, make it available for garbage collection del data, imagecache # Match media library items with any cached artwork matchTextures(mediatype, mediaitems, jcomms, database, force, nodownload) if force: ITEMLIMIT = 0 elif nodownload: ITEMLIMIT = -1 else: ITEMLIMIT = 100 # Count number of items that need to be downloaded, and # output details of items that are stale or missing from the cache (unless force, # in which case everything is being downloaded so don't output anything!) itemCount = 0 for item in mediaitems: if item.status in [MyMediaItem.STATUS_MISSING, MyMediaItem.STATUS_STALE]: itemCount += 1 if ITEMLIMIT == -1 or itemCount < ITEMLIMIT: reason = "Need to cache" if item.status == MyMediaItem.STATUS_MISSING else "Cache stale " MSG = "%s: [%-10s] for %s: %s\n" % (reason, item.itype.center(10), re.sub("(.*)s$", "\\1", item.mtype), item.getFullName()) gLogger.out(MSG) elif itemCount == ITEMLIMIT: gLogger.out("...and many more! (First %d items shown)\n" % ITEMLIMIT) if nodownload: TOTALS.addNotCached() for item in mediaitems: if item.status == MyMediaItem.STATUS_MISSING: TOTALS.bump("Not in Cache", item.itype) # Don't proceed beyond this point unless there is something to download... if itemCount == 0 or nodownload: return # Queue up the items to be downloaded... single_work_queue = Queue.Queue() multiple_work_queue = Queue.Queue() error_queue = Queue.Queue() complete_queue = Queue.Queue() gLogger.out("\n") # Identify unique itypes, so we can group items in the queue # This is crucial to working out when the first/last item is loaded # in order to calculate accurate elapsed times by itype unique_items = {} for item in mediaitems: if not item.itype in unique_items: unique_items[item.itype] = True if force and gConfig.DOWNLOAD_PREDELETE: TOTALS.TimeStart(mediatype, "PreDelete") TOTALS.init() dbitems = 0 for item in mediaitems: if item.dbid != 0: dbitems += 1 dbitem = 0 with database: for ui in sorted(unique_items): for item in mediaitems: if item.dbid != 0 and item.itype == ui: dbitem += 1 gLogger.progress("Pre-deleting cached items %d of %d... rowid %d, cachedurl %s" % (dbitem, dbitems, item.dbid, item.cachedurl)) TOTALS.start(item.mtype, item.itype) database.deleteItem(item.dbid, item.cachedurl) TOTALS.bump("Deleted", item.itype) TOTALS.finish(item.mtype, item.itype) item.dbid = 0 item.cachedurl = "" TOTALS.stop() TOTALS.TimeEnd(mediatype, "PreDelete") gLogger.progress("") c = sc = mc = 0 for ui in sorted(unique_items): for item in mediaitems: if item.status in [MyMediaItem.STATUS_MISSING, MyMediaItem.STATUS_STALE] and item.itype == ui: c += 1 isSingle = False if gConfig.SINGLETHREAD_URLS: for site in gConfig.SINGLETHREAD_URLS: if site.search(item.decoded_filename): sc += 1 if gLogger.VERBOSE and gLogger.LOGGING: gLogger.log("QUEUE ITEM: single [%s], %s" % (site.pattern, item)) single_work_queue.put(item) item.status = MyMediaItem.STATUS_QUEUED isSingle = True break if not isSingle: mc += 1 if gLogger.VERBOSE and gLogger.LOGGING: gLogger.log("QUEUE ITEM: %s" % item) multiple_work_queue.put(item) item.status = MyMediaItem.STATUS_QUEUED gLogger.progress("Queueing work item: single thread %d, multi thread %d" % (sc, mc), every=50, finalItem=(c==itemCount)) # Don't need this data anymore, make it available for garbage collection del mediaitems TOTALS.TimeStart(mediatype, "Download") THREADS = [] if not single_work_queue.empty(): gLogger.log("Creating 1 download thread for single access sites") t = MyImageLoader(single_work_queue, multiple_work_queue, error_queue, complete_queue, gConfig, gLogger, TOTALS, force, gConfig.DOWNLOAD_RETRY) THREADS.append(t) t.setDaemon(True) if not multiple_work_queue.empty(): tCount = gConfig.DOWNLOAD_THREADS["download.threads.%s" % mediatype] THREADCOUNT = tCount if tCount <= mc else mc gLogger.log("Creating %d download thread(s) for multi-access sites" % THREADCOUNT) for i in range(THREADCOUNT): t = MyImageLoader(multiple_work_queue, single_work_queue, error_queue, complete_queue, gConfig, gLogger, TOTALS, force, gConfig.DOWNLOAD_RETRY) THREADS.append(t) t.setDaemon(True) # Start the threads... for t in THREADS: t.start() threadcount = len(THREADS) updateInterval = 1.0 itemsRemaining = itemCount perfhistory = [] showProgress(threadcount, itemCount, single_work_queue.qsize(), multiple_work_queue.qsize(), error_queue.qsize(), itemsRemaining) while threadcount > 0: pace = time.time() completed = 0 while True: try: qItem = complete_queue.get(block=True, timeout=updateInterval) complete_queue.task_done() if qItem is None: threadcount -= 1 else: completed += 1 itemsRemaining -= 1 if (time.time() - pace) >= updateInterval or threadcount <= 0: break except Queue.Empty: break showProgress(threadcount, itemCount, single_work_queue.qsize(), multiple_work_queue.qsize(), error_queue.qsize(), itemsRemaining, completed, time.time() - pace, perfhistory) TOTALS.TimeEnd(mediatype, "Download") gLogger.progress("", newLine=True, noBlank=True) if not error_queue.empty(): gLogger.out("\nThe following items could not be downloaded:\n\n") while not error_queue.empty(): item = error_queue.get() error_queue.task_done() # Ignore itypes with a period, eg. "cast.thumb" or "season.banner" if item.mtype in ["sets", "movies", "tvshows", "seasons", "episodes"] and item.itype.find(".") == -1: drop_id = "%s#%d" % (item.mtype, item.libraryid) if drop_id not in drop_items: drop_items[drop_id] = {"libraryid": item.libraryid, "title": item.name, "type": item.getTypeSingular(), "items": {}} drop_item = drop_items[drop_id] artwork_items = drop_item["items"] artwork_items["art.%s" % item.itype] = "" drop_item["items"] = artwork_items drop_items[drop_id] = drop_item name = addEllipsis(50, item.getFullName()) gLogger.out("[%-10s] [%-40s] %s\n" % (item.itype, name, item.decoded_filename)) gLogger.log("ERROR ITEM: %s" % item) def dump_drop_items(drop_items): if gConfig.CACHE_DROP_INVALID_FILE: outfile = codecs.open(gConfig.CACHE_DROP_INVALID_FILE, "wb", encoding="utf-8") outfile.write(json.dumps(drop_items.values(), indent=2, ensure_ascii=True, sort_keys=True)) outfile.close() def showProgress(tRunning, maxItems, swqs, mwqs, ewqs, remaining=0, completed=0, interval=0.0, history=None): c = 0 i = 0.0 # Apply a smoothing function based on fixed number of previous samples if history is not None and interval != 0.0: history.insert(0, (completed, interval)) if len(history) > 15: history.pop() for p in history: c += p[0] i += p[1] if i != 0.0: # Smoothed tpersec tpersec = c / i if i != 0.0 else 0.0 # Instantaneous tpersec - if sub-second don't normalise to a second as it results in erroneous value if interval < 1.0: itpersec = completed else: itpersec = completed / interval if interval != 0.0 else 0.0 else: tpersec = itpersec = 0.0 # Accumulate per-second download stats TOTALS.MCOUNT += completed TOTALS.MAVG += interval if TOTALS.MMIN == 0 or TOTALS.MMIN > itpersec: TOTALS.MMIN = itpersec if TOTALS.MMAX < itpersec: TOTALS.MMAX = itpersec eta = TOTALS.secondsToTime(remaining / tpersec, withMillis=False) if tpersec != 0.0 else "**:**:**" msg = " (%05.2f downloads per second, ETA: %s)" % (itpersec, eta) gLogger.progress("Caching artwork: %d item%s remaining of %d (qs: %d, qm: %d), %d error%s, %d thread%s active%s" % \ (remaining, "s"[remaining==1:], maxItems, swqs, mwqs, ewqs, "s"[ewqs==1:], tRunning, "s"[tRunning==1:], msg)) def matchTextures(mediatype, mediaitems, jcomms, database, force, nodownload): TOTALS.TimeStart(mediatype, "Compare") if gConfig.CHUNKED: matchTextures_chunked(mediatype, mediaitems, jcomms, database, force, nodownload) else: matchTextures_fast(mediatype, mediaitems, jcomms, database, force, nodownload) TOTALS.TimeEnd(mediatype, "Compare") return def matchTextures_fast(mediatype, mediaitems, jcomms, database, force, nodownload): gLogger.progress("Loading Textures DB...") dbfiles = {} with database: for r in database.getRows(allfields=False): dbfiles[r["url"]] = r gLogger.log("Loaded %d items from texture cache database" % len(dbfiles)) gLogger.progress("Matching library and texture items...") itemCount = 0 for item in mediaitems: dbrow = dbfiles.get(item.decoded_filename, None) matchTextures_item_row(mediatype, jcomms, item, dbrow, force, nodownload) # Don't need this data anymore, make it available for garbage collection del dbfiles gLogger.progress("") return def matchTextures_chunked(mediatype, mediaitems, jcomms, database, force, nodownload): ITEMLIMIT = -1 if nodownload else 100 unmatched = len(mediaitems) matched = 0 skipped = 0 # Build a URL based hash of indexes so that we can quickly access mediaitems # by index for a given URL url_to_index = {} for inum, item in enumerate(mediaitems): url_to_index[item.decoded_filename] = inum dbindex = 0 dbmax = 0 with database: folders = database.getTextureFolders() for fnum, folder in enumerate(folders): # Once all library items have been matched, no need to continue querying textures DB if unmatched == 0: break gLogger.progress("Loading Textures DB: chunk %2d of %d [unmatched %d: matched %d, skipped %d] (%d of %d)" % (fnum+1, len(folders), unmatched, matched, skipped, dbindex, dbmax)) dbfiles = [] for r in database.getRows(database.getTextureFolderFilter(folder), allfields=False): dbfiles.append(r) dbindex = 0 dbmax = len(dbfiles) for dbrow in dbfiles: dbindex += 1 gLogger.progress("Loading Textures DB: chunk %2d of %d [unmatched %d: matched %d, skipped %d] (%d of %d)" % (fnum+1, len(folders), unmatched, matched, skipped, dbindex, dbmax), every=50, finalItem=(dbindex==dbmax)) inum = url_to_index.get(dbrow["url"], None) if inum is not None: item = mediaitems[inum] if item.status == MyMediaItem.STATUS_UNKNOWN: unmatched -= 1 matchTextures_item_row(mediatype, jcomms, item, dbrow, force, nodownload) if item.status == MyMediaItem.STATUS_IGNORE: skipped += 1 else: matched += 1 # Any media library items that haven't been matched must also be processed for item in mediaitems: if item.status == MyMediaItem.STATUS_UNKNOWN: matchTextures_item_row(mediatype, jcomms, item, None, force, nodownload) gLogger.progress("") return def matchTextures_item_row(mediatype, jcomms, item, dbrow, force, nodownload): if item.mtype == "tvshows" and item.season == "Season All": TOTALS.bump("Season-all", item.itype) # Don't need to cache file if it's already in the cache, unless forced... # Assign the texture cache database id and cachedurl so that removal will avoid having # to retrieve these items from the database. if dbrow: if force: if gConfig.cache_refresh_date: if MyUtility.is_cache_item_stale(gConfig, jcomms, item): item.status = MyMediaItem.STATUS_STALE else: item.status = MyMediaItem.STATUS_IGNORE else: item.status = MyMediaItem.STATUS_MISSING if item.status != MyMediaItem.STATUS_IGNORE: item.dbid = dbrow["textureid"] item.cachedurl = dbrow["cachedurl"] else: TOTALS.bump("Skipped", item.itype) else: if nodownload and MyUtility.is_cache_item_stale(gConfig, jcomms, item): item.status = MyMediaItem.STATUS_STALE TOTALS.bump("Stale Item", item.itype) else: item.status = MyMediaItem.STATUS_IGNORE TOTALS.bump("Skipped", item.itype) if gLogger.VERBOSE and gLogger.LOGGING: gLogger.log("ITEM SKIPPED: %s" % item) else: item.status = MyMediaItem.STATUS_MISSING return # # Iterate over all the elements, seeking out artwork to be stored in a list. # Use recursion to process season and episode sub-elements. # def parseURLData(jcomms, mediatype, mediaitems, imagecache, data, title_name, id_name, showName=None, season=None, pvrGroup=None): gLogger.reset() SEASON_ALL = (showName is not None and season is None) for item in data: if title_name in item: title = item[title_name] if showName: mediatype = "tvshows" name = showName if season: episode = re.sub("([0-9]*x[0-9]*)\..*", "\\1", title) longName = "%s, %s Episode %s" % (showName, season, episode) else: season = title episode = None longName = "%s, %s" % (showName, title) elif pvrGroup: name = pvrGroup season = title longName = "%s, %s" % (pvrGroup, title) episode = None # channelid may be missing for some reason item["channelid"] = item.get("channelid",0) else: name = title longName = name season = item.get("artist", None) if title_name != "artist" else None episode= item.get("album", None) if title_name != "album" else None gLogger.progress("Parsing %s: %s..." % (mediatype.capitalize(), longName), every = 25) for a in ["fanart", "poster", "thumb", "thumbnail"]: if a in item and evaluateURL(a, item[a], imagecache): mediaitems.append(MyMediaItem(mediatype, a, name, season, episode, item[a], 0, None, item[id_name], False)) if "art" in item: if SEASON_ALL and "poster" in item["art"]: SEASON_ALL = False (poster_url, fanart_url, banner_url) = jcomms.getSeasonAll(item["art"]["poster"]) if poster_url and evaluateURL("poster", poster_url, imagecache): mediaitems.append(MyMediaItem(mediatype, "poster", name, "Season All", None, poster_url, 0, None, item[id_name], False)) if fanart_url and evaluateURL("fanart", fanart_url, imagecache): mediaitems.append(MyMediaItem(mediatype, "fanart", name, "Season All", None, fanart_url, 0, None, item[id_name], False)) if banner_url and evaluateURL("banner", banner_url, imagecache): mediaitems.append(MyMediaItem(mediatype, "banner", name, "Season All", None, banner_url, 0, None, item[id_name], False)) for a in item["art"]: imgtype_short = a.replace("tvshow.","") if evaluateURL(imgtype_short, item["art"][a], imagecache): mediaitems.append(MyMediaItem(mediatype, imgtype_short, name, season, episode, item["art"][a], 0, None, item[id_name], False)) if "cast" in item: for a in item["cast"]: if "thumbnail" in a and evaluateURL("cast.thumb", a["thumbnail"], imagecache): mediaitems.append(MyMediaItem(mediatype, "cast.thumb", a["name"], name, None, a["thumbnail"], 0, None, item[id_name], False)) if mediatype in ["artists", "albums", "movies", "tags", "tvshows"]: for file in jcomms.getExtraArt(item): if evaluateURL(file["type"], file["file"], imagecache): mediaitems.append(MyMediaItem(mediatype, file["type"], name, season, episode, file["file"], 0, None, item[id_name], False)) if "seasons" in item: parseURLData(jcomms, "seasons", mediaitems, imagecache, item["seasons"], "label", "season", showName=title) elif "episodes" in item: parseURLData(jcomms, "episodes", mediaitems, imagecache, item["episodes"], "label", "episodeid", showName=showName, season=title) season = None elif "channels" in item: parseURLData(jcomms, "%s.channel" % mediatype, mediaitems, imagecache, item["channels"], "channel", "channelid", pvrGroup=title) elif "genres" in item: parseURLData(jcomms, "genres", mediaitems, imagecache, item["genres"], "label", "genreid", showName=title) # Include or exclude URL depending on basic properties - has it # been "seen" before (in which case, discard as no point caching # it twice. Or discard if matches an "ignore" rule. # # Otherwise include it, and add it to the imagecache so it can # be excluded in future if "seen" again. # def evaluateURL(imgtype, url, imagecache): if not url or url == "": TOTALS.bump("Undefined", imgtype) imagecache[""] += 1 return False if gConfig.CACHE_ARTWORK and imgtype not in gConfig.CACHE_ARTWORK: if gLogger.LOGGING: decoded_url = MyUtility.normalise(url, strip=True) gLogger.log("Ignored [%-12s] for [%s] as image type not in cache.artwork list" % (imgtype, decoded_url)) TOTALS.bump("Ignored", imgtype) imagecache[url] = 1 return False if url in imagecache: TOTALS.bump("Duplicate", imgtype) imagecache[url] += 1 return False if gConfig.CACHE_IGNORE_TYPES: decoded_url = MyUtility.normalise(url, strip=True) for ignore in gConfig.CACHE_IGNORE_TYPES: if ignore.search(decoded_url): gLogger.log("Ignored [%-12s] image due to [%s] rule: %s" % (imgtype, ignore.pattern, decoded_url)) TOTALS.bump("Ignored", imgtype) imagecache[url] = 1 return False imagecache[url] = 0 return True def qaData(mediatype, jcomms, database, data, title_name, id_name, rescan, work=None, mitems=None, showName=None, season=None, pvrGroup=None): gLogger.reset() if mitems is None: TOTALS.TimeStart(mediatype, "Parse") workItems= {} mediaitems = [] else: workItems = work mediaitems = mitems zero_items = [] blank_items = [] art_items = [] check_file = False nfo_file = False if mediatype in ["movies", "tags", "episodes"]: check_file = gConfig.QA_FILE nfo_file = (gConfig.qa_nfo_refresh_date is not None) zero_items.extend(gConfig.getQAFields("zero", mediatype, stripModifier=False)) blank_items.extend(gConfig.getQAFields("blank", mediatype, stripModifier=False)) art_items.extend(gConfig.getQAFields("art", mediatype, stripModifier=False)) #Hack to prevent top level genre group items (movie, tvshow, musicvideo) being #reported as having missing artwork (since they don't have any artwork). if mediatype == "vgenres" and not showName: zero_items = blank_items = art_items = [] for item in data: title = item.get(title_name, "") libraryid = item.get(id_name, 0) if showName: if season: episode = re.sub("([0-9]*x[0-9]*)\..*", "\\1", title) name = "%s, %s Episode %s" % (showName, season, episode) else: episode = None name = "%s, %s" % (showName, title) elif pvrGroup: name = "%s, %s" % (pvrGroup, title) else: name = title season = None episode = None gLogger.progress("Parsing %s: %s..." % (mediatype.capitalize(), name), every = 25) missing = {} for i in zero_items: (j, MOD, MOD_MISSING_SILENT, MOD_MISSING_WARN_FAIL) = splitModifierToken(i) ismissing = True if j in item: ismissing = (item[j] == 0) if ismissing and not MOD_MISSING_SILENT: missing["zero %s" % j] = MOD_MISSING_WARN_FAIL for i in blank_items: (j, MOD, MOD_MISSING_SILENT, MOD_MISSING_WARN_FAIL) = splitModifierToken(i) ismissing = True if j in item: if type(item[j]) is dict: # Example dict: streamdetails for field in item[j]: if item[j][field]: ismissing = False break else: ismissing = (item[j] == "" or item[j] == [] or item[j] == [""]) if ismissing and not MOD_MISSING_SILENT: missing["missing %s" % j] = MOD_MISSING_WARN_FAIL for i in art_items: (j, MOD, MOD_MISSING_SILENT, MOD_MISSING_WARN_FAIL) = splitModifierToken(i) if "art" in item: artwork = item.get("art", {}).get(j, "") else: artwork = item.get(j, "") if artwork == "": if not MOD_MISSING_SILENT: if MOD_MISSING_WARN_FAIL: if gConfig.QA_FAIL_CHECKEXISTS and "file" in item: if qa_check_artfile_exists(jcomms, mediatype, item, i): missing["missing %s, local is available" % j] = True else: missing["missing %s, local not found" % j] = gConfig.QA_FAIL_MISSING_LOCAL_ART else: missing["missing %s" % j] = True else: missing["missing %s" % j] = False else: decoded_url = MyUtility.normalise(artwork, strip=True) FAILED = False if gConfig.QA_FAIL_TYPES and MOD_MISSING_WARN_FAIL: for qafailtype in gConfig.QA_FAIL_TYPES: if qafailtype.search(decoded_url): if gConfig.QA_FAIL_CHECKEXISTS and "file" in item: if qa_check_artfile_exists(jcomms, mediatype, item, i): missing["URL %s %s, local is available" % (j, qafailtype.pattern)] = True else: missing["URL %s %s, local not found" % (j, qafailtype.pattern)] = gConfig.QA_FAIL_MISSING_LOCAL_ART else: missing["URL %s %s" % (j, qafailtype.pattern)] = True FAILED = True break if not FAILED and gConfig.QA_WARN_TYPES: for qawarntype in gConfig.QA_WARN_TYPES: if qawarntype.search(decoded_url): missing["URL %s %s" % (j, qawarntype.pattern)] = False break if (check_file or nfo_file) and "file" in item: files = None file_not_found = check_file nfo_not_found = nfo_file for file in MyUtility.unstackFiles(item["file"], addcombinedfile=True): dir = os.path.dirname(file) data = jcomms.getDirectoryList(dir, mediatype="files", properties=["file", "lastmodified"]) if files == None: files = data.get("result", {}).get("files", []) if check_file and file_not_found and [f for f in files if f["filetype"] == "file" and f.get("file", None) == file]: file_not_found = False if nfo_file and nfo_not_found: nfofile = "%s.nfo" % os.path.splitext(file)[0] for f in [x for x in files if x["filetype"] == "file" and x.get("file", None) == nfofile]: nfo_not_found = False jcomms.setTimeStamp(f) if "lastmodified_timestamp" in f and \ f["lastmodified_timestamp"] >= gConfig.qa_nfo_refresh_date: missing["modified nfo"] = True if file_not_found: missing["missing file"] = False if nfo_not_found: missing["missing nfo"] = False if "seasons" in item: qaData("seasons", jcomms, database, item["seasons"], "label", "season", False, \ work=workItems, mitems=mediaitems, showName=title) if "episodes" in item: qaData("episodes", jcomms, database, item["episodes"], "label", "episodeid", False, \ work=workItems, mitems=mediaitems, showName=showName, season=title) season = None if "channels" in item: qaData("%s.channel" % mediatype, jcomms, database, item["channels"], "channel", "channelid", False, \ work=workItems, mitems=mediaitems, pvrGroup=title) if "genres" in item: qaData(mediatype, jcomms, database, item["genres"], "label", "genreid", False, \ work=workItems, mitems=mediaitems, showName=title) if missing != {}: if mediatype.startswith("pvr.") or mediatype in ["agenres", "vgenres"]: mtype = mediatype else: mtype = mediatype[:-1].capitalize() if mtype == "Tvshow": mtype = "TVShow" if "file" in item: mFAIL = "; ".join([x for x in missing if missing[x] == True]) mWARN = "; ".join([x for x in missing if missing[x] == False]) else: mFAIL = "" mWARN = "; ".join(missing) msg = "" if mFAIL: msg = "%sFAIL (%s), " % (msg, mFAIL) if mWARN: msg = "%sWARN (%s), " % (msg, mWARN) msg = msg [:-2] mediaitems.append("%-8s [%-50s]: %s" % (mtype, addEllipsis(50, name), msg)) if "file" in item and mFAIL: dir = "%s;%s" % (mediatype, os.path.dirname(MyUtility.unstackFiles(item["file"])[0])) libraryids = workItems[dir] if dir in workItems else [] libraryids.append({"id": libraryid, "name": name}) workItems[dir] = libraryids if mitems is None: TOTALS.TimeEnd(mediatype, "Parse") gLogger.progress("") for m in mediaitems: gLogger.out("%s\n" % m) if rescan and mediatype in ["movies", "tags", "sets", "tvshows"]: TOTALS.TimeStart(mediatype, "Rescan") jcomms.rescanDirectories(workItems) TOTALS.TimeEnd(mediatype, "Rescan") # Return True if an artwork item can be matched, this means we can # FAIL the item and remove/re-scrape. If no artwork exists, then just # WARN because removing/rescraping won't serve any purpose. def qa_check_artfile_exists(jcomms, mediatype, item, artwork): if "file" not in item: return False filename = MyUtility.unstackFiles(item["file"], addcombinedfile=True)[0] dir = os.path.dirname(filename) data = jcomms.getDirectoryList(dir, mediatype="files", properties=["file", "lastmodified"]) files = data.get("result", {}).get("files", []) if files: for art in get_qa_artworkcandidates(mediatype, filename, item, artwork): for file in files: if file["filetype"] == "file" and file["file"] == art: return True return False # Construct a list of potential artwork candidates # based on file name and artwork type. def get_qa_artworkcandidates(mediatype, filename, item, artwork): art = [] types = [] fname = os.path.splitext(filename)[0] parent = os.path.dirname(filename) fs_bs = "\\" if filename.find("\\") != -1 else "/" types.append(artwork) if artwork == "poster": types.append("thumb") elif artwork == "clearlogo": types.append("logo") elif artwork == "discart": types.append("disc") for t in types: art.append("%s-%s.jpg" % (fname, t)) art.append("%s-%s.png" % (fname, t)) art.append("%s%s%s.jpg" % (parent, fs_bs, t)) art.append("%s%s%s.png" % (parent, fs_bs, t)) if mediatype in ["albums", "songs"] and t == "thumbnail": art.append("%s.jpg" % (fname)) art.append("%s.png" % (fname)) art.append("%s%s%s.jpg" % (parent, fs_bs, "folder")) art.append("%s%s%s.png" % (parent, fs_bs, "folder")) art.append("%s%s%s.jpg" % (parent, fs_bs, "cover")) art.append("%s%s%s.png" % (parent, fs_bs, "cover")) return art def splitModifierToken(field): if field and field[:1] in ["?", "#", "!"]: m = field[:1] return (field[1:], m, (m in ["#", "!"]), (False if (m == "?" or m == "#") else True)) else: return (field, "", False, True) def missingFiles(mediatype, data, fileList, title_name, id_name, showName=None, season=None): gLogger.reset() if showName is None: TOTALS.TimeStart(mediatype, "Parse") for item in data: libraryid = item[id_name] if title_name in item: title = item[title_name] if showName: name = showName if season: episode = re.sub("([0-9]*x[0-9]*)\..*", "\\1", title) name = "%s, %s Episode %s" % (showName, season, episode) else: season = title episode = None name = "%s, %s" % (showName, season) else: name = title season = None episode = None gLogger.progress("Parsing %s: %s..." % (mediatype.capitalize(), name), every = 25) # Remove matched file from fileList - what files remain at the end # will be reported to the user if mediatype != "tvshows" and "file" in item: for file in MyUtility.unstackFiles(item["file"]): try: fileList.remove(file) except ValueError: pass if "seasons" in item: missingFiles("seasons", item["seasons"], fileList, "label", "season", showName=title) if "episodes" in item: missingFiles("episodes", item["episodes"], fileList, "label", "episodeid", showName=showName, season=title) season = None if showName is None: TOTALS.TimeEnd(mediatype, "Parse") gLogger.progress("") if fileList != []: gLogger.out("The following media files are not present in the \"%s\" media library:\n\n" % mediatype) for file in fileList: gLogger.out("%s\n" % file) def queryLibrary(mediatype, query, data, title_name, id_name, work=None, mitems=None, showName=None, season=None, pvrGroup=None): gLogger.reset() if mitems is None: TOTALS.TimeStart(mediatype, "Parse") workItems= {} mediaitems = [] else: workItems = work mediaitems = mitems fields, tuples = parseQuery(query) for item in data: libraryid = item.get(id_name, 0) if id_name == "songid": if title_name in item and "artist" in item: title = "%s (%s)" % (item[title_name], "/".join(item["artist"])) else: if title_name in item: title = item[title_name] if showName: if season: episode = re.sub("([0-9]*x[0-9]*)\..*", "\\1", title) name = "%s, %s Episode %s" % (showName, season, episode) else: episode = None name = "%s, %s" % (showName, title) elif pvrGroup: name = "%s, %s" % (pvrGroup, title) else: name = title season = None episode = None gLogger.progress("Parsing %s: %s..." % (mediatype.capitalize(), name), every = 25) RESULTS = [] try: for field, field_split, condition, inverted, value, logic in tuples: temp = item for f in field_split: temp = searchItem(temp, f) if temp is None: break if temp is not None: if type(temp) is list: for t in temp: MATCHED = evaluateCondition(t, condition, value) if inverted: MATCHED = not MATCHED if MATCHED: break matched_value = ", ".join(str(x) for x in temp) else: if isinstance(temp, basestring): temp = MyUtility.normalise(temp, strip=True) MATCHED = evaluateCondition(temp, condition, value) if inverted: MATCHED = not MATCHED matched_value = temp else: MATCHED = False matched_value = None RESULTS.append([MATCHED, logic, field, matched_value]) except: pass MATCHED = False FIELDS = [] DISPLAY = "" for matched, logic, field, value in RESULTS: if logic == "and": if matched == False: MATCHED = False elif logic == "or": if matched == True: MATCHED = True elif logic is None: MATCHED = matched else: MATCHED = False # Only output each field value once... if not field in FIELDS: FIELDS.append(field) try: throw_exception = value + 1 DISPLAY = "%s, %s = %s" % (DISPLAY, field, value) except: DISPLAY = "%s, %s = \"%s\"" % (DISPLAY, field, value) if MATCHED: mediaitems.append([name, DISPLAY[2:]]) if "seasons" in item: queryLibrary("seasons", query, item["seasons"], "label", "season", \ work=workItems, mitems=mediaitems, showName=title) if "episodes" in item: queryLibrary("episodes", query, item["episodes"], "label", "episodeid", \ work=workItems, mitems=mediaitems, showName=showName, season=title) season = None if "channels" in item: queryLibrary("%s.channel" % mediatype, query, item["channels"], "channel", "channelid", \ work=workItems, mitems=mediaitems, pvrGroup=title) if "genres" in item: queryLibrary("genres", query, item["genres"], "label", "genreid", \ work=workItems, mitems=mediaitems, showName=title) if mitems is None: TOTALS.TimeEnd(mediatype, "Parse") gLogger.progress("") for m in mediaitems: gLogger.out("Matched: [%-50s] %s" % (addEllipsis(50, m[0]), m[1]), newLine=True) def addEllipsis(maxlen, aStr): if len(aStr) <= maxlen: return aStr ileft = int(maxlen/2) - 2 iright = int(maxlen/2) - 1 return "%s...%s" % (aStr[0:ileft], aStr[-iright:]) def searchItem(data, field): if field in data: return data[field] if type(data) is list: tList = [] for item in data: value = searchItem(item, field) if value: tList.append(value) return tList return None def evaluateCondition(input, condition, value): if type(input) is int: value = int(value) if type(input) is float: value = float(value) if condition in ["=", "=="]: return (input == value) elif condition == "!=": return (input != value) elif condition == ">": return (input > value) elif condition == "<": return (input < value) elif condition == ">=": return (input >= value) elif condition == "<=": return (input <= value) elif condition == "contains": return (input.find(value) != -1) elif condition == "startswith": return (input.startswith(value)) elif condition == "endswith": return (input.endswith(value)) return False def parseQuery(query): condition = ["==", "=", "!=", ">", ">=", "<", "<=", "contains", "startswith", "endswith"] logic = ["and", "or"] fields = [] tuples = [] FIELDNAME_NEXT = True tField = tValue = tCondition = tLogic = None query = MyUtility.toUnicode(query) newValue = "" IN_STR=False for value in query: if value == "'" or value == '"': IN_STR = not IN_STR if value == " " and IN_STR: newValue = "%s\t" % newValue else: newValue = "%s%s" % (newValue, value) INVERT=False for value in newValue.split(" "): if value == "": continue value_lower = value.lower() if value_lower == "not": INVERT=True continue #and, or etc. if value_lower in logic: FIELDNAME_NEXT=True tLogic = value # ==, >=, contains etc. elif value_lower in condition: if value == "=": value = "==" FIELDNAME_NEXT=False tCondition = value #Value elif not FIELDNAME_NEXT: if value.startswith("'") or value.startswith('"'): value = value[1:] if value.endswith("'") or value.endswith('"'): value = value[:-1] FIELDNAME_NEXT=True tValue = value.replace("\t", " ") tuples.append([tField, tField.split("."), tCondition, INVERT, tValue, tLogic]) INVERT=False #Field name else: tField = value_lower fields.append(tField.split(".")[0]) FIELDNAME_NEXT=False return ",".join(fields), tuples def watchedWrite(filename, mediaitems): MYLIST = [] for m in mediaitems: MYLIST.append({"type": m.mtype, "name": m.name, "episode_year": m.episode_year, "playcount": m.playcount, "lastplayed": m.lastplayed, "resume": m.resume}) try: OUTPUTFILE = codecs.open(filename, "wb", encoding="utf-8") OUTPUTFILE.write(json.dumps(MYLIST, indent=2, ensure_ascii=True)) OUTPUTFILE.close() except: gLogger.out("ERROR: failed to write the watched list to file [%s]" % filename, newLine=True) def watchedRead(filename, mediaitems): BUFFER = "" try: INPUTFILE = codecs.open(filename, "rb", encoding="utf-8") BUFFER = INPUTFILE.read() INPUTFILE.close() MYLIST = json.loads(BUFFER) for m in MYLIST: mediakey = "%s;%s;%s" % (m["type"], m["name"], m["episode_year"]) mediaitems[mediakey] = MyWatchedItem(m["type"], m["name"], m["episode_year"], m["playcount"], m["lastplayed"], m["resume"]) except: gLogger.out("ERROR: failed to read the watched list from file [%s]" % filename, newLine=True) return False return True def watchedBackup(mediatype, filename, data, title_name, id_name, work=None, mitems=None, showName=None, season=None): gLogger.reset() if mitems is None: TOTALS.TimeStart(mediatype, "Parse") workItems= {} mediaitems = [] else: workItems = work mediaitems = mitems for item in data: libraryid = item.get(id_name, 0) title = item.get(title_name, "") if showName: shortName = showName if season: episode_year = re.sub("([0-9]*x[0-9]*)\..*", "\\1", title) longName = "%s, %s Episode %s" % (showName, season, episode_year) else: episode_year = None longName = "%s, %s" % (showName, title) else: season = None episode_year = item.get("year", 0) longName = shortName = title gLogger.progress("Parsing %s: %s..." % (mediatype.capitalize(), longName), every = 25) playcount = item.get("playcount", 0) lastplayed = item.get("lastplayed", "") resume = item.get("resume", {"position": 0.0, "total": 0.0}) if playcount != 0 or lastplayed != "" or resume["position"] != 0.0: mediaitems.append(MyWatchedItem(mediatype, shortName, episode_year, playcount, lastplayed, resume)) if "seasons" in item: watchedBackup("seasons", filename, item["seasons"], "label", "season", \ work=workItems, mitems=mediaitems, showName=title) if "episodes" in item: watchedBackup("episodes", filename, item["episodes"], "label", "episodeid", \ work=workItems, mitems=mediaitems, showName=showName, season=title) season = None if mitems is None: TOTALS.TimeEnd(mediatype, "Parse") gLogger.progress("") watchedWrite(filename, mediaitems) def watchedRestore(mediatype, jcomms, filename, data, title_name, id_name, work=None, mitems=None, showName=None, season=None): gLogger.reset() if mitems is None: TOTALS.TimeStart(mediatype, "Parse") workItems= {} mediaitems = {} if not watchedRead(filename, mediaitems): return else: workItems = work mediaitems = mitems for item in data: libraryid = item.get(id_name, 0) title = item.get(title_name, "") if showName: shortName = showName if season: episode_year = re.sub("([0-9]*x[0-9]*)\..*", "\\1", title) longName = "%s, %s Episode %s" % (showName, season, episode_year) else: episode_year = None longName = "%s, %s" % (showName, title) else: season = None episode_year = item.get("year", 0) longName = shortName = title gLogger.progress("Parsing %s: %s..." % (mediatype.capitalize(), longName), every = 25) playcount = item.get("playcount", 0) lastplayed = item.get("lastplayed", "") resume = item.get("resume", {"position": 0.0, "total": 0.0}) if mediatype in ["movies", "episodes"]: mediakey = "%s;%s;%s" % (mediatype, shortName, episode_year) if mediakey in mediaitems: m = mediaitems[mediakey] if m.libraryid == 0: m.libraryid = libraryid # Update watched object with latest library values unless overwriting, # in which case keep the values that are being restored. if not gConfig.WATCHEDOVERWRITE: m.refresh(gConfig.JSON_HAS_SETRESUME, playcount, lastplayed, resume) # Set update state based on old/new values m.setState(gConfig.JSON_HAS_SETRESUME, playcount, lastplayed, resume) if "seasons" in item: watchedRestore("seasons", jcomms, filename, item["seasons"], "label", "season", \ work=workItems, mitems=mediaitems, showName=title) if "episodes" in item: watchedRestore("episodes", jcomms, filename, item["episodes"], "label", "episodeid", \ work=workItems, mitems=mediaitems, showName=showName, season=title) season = None if mitems is None: TOTALS.TimeEnd(mediatype, "Parse") gLogger.progress("") RESTORED = UNCHANGED = UNMATCHED = ERROR = 0 for mediakey in sorted(mediaitems): m = mediaitems[mediakey] shortName = "%s, Episode %s" % (m.name, m.episode_year) if m.mtype == "episodes" else "%s (%s)" % (m.name, m.episode_year) if m.libraryid == 0: gLogger.out("NO MATCH %s: %s" % (m.mtype[:-1], shortName), newLine = True, log = True) UNMATCHED += 1 else: if m.state != 0: UNCHANGED += 1 elif watchedItemUpdate(jcomms, m, shortName): gLogger.out("Restored %s: %s" % (m.mtype[:-1], shortName), newLine = True, log = True) RESTORED += 1 else: gLogger.out("FAILED %s: %s" % (m.mtype[:-1], shortName), newLine = True, log = True) ERROR += 1 gLogger.out("", newLine=True) gLogger.out("Watched List item summary: restored %d, unchanged %d, unmatched %d, failed %d\n" % (RESTORED, UNCHANGED, UNMATCHED, ERROR), newLine=True) def watchedItemUpdate(jcomms, mediaitem, shortName): if mediaitem.mtype == "movies": method = "VideoLibrary.SetMovieDetails" mediaid = "movieid" else: method = "VideoLibrary.SetEpisodeDetails" mediaid = "episodeid" REQUEST = {"method": method, "params": {mediaid: mediaitem.libraryid, "playcount": mediaitem.playcount, "lastplayed": mediaitem.lastplayed }} if gConfig.JSON_HAS_SETRESUME: REQUEST["params"]["resume"] = {"position": mediaitem.resume["position"]} gLogger.progress("Restoring %s: %s..." % (mediaitem.mtype[:-1], shortName)) data = jcomms.sendJSON(REQUEST, "libWatchedList", checkResult=False) return ("result" in data and data["result"] == "OK") def duplicatesList(mediatype, jcomms, data): imdblist = [] dupelist = [] # Iterate over movies, building up list of IMDb numbers # for movies that appear more than once... for movie in data: imdb = movie["imdbnumber"] if imdb: if imdb in imdblist: if not imdb in dupelist: dupelist.append(imdb) else: imdblist.append(imdb) # Iterate over the list of duplicate IMDb numbers, # and build up a list of matching movie details. # # dupelist will be in ascending alphabetical order based # on the title name of the first movie to be matched # above (the JSON data is in title order). # # Sort the list of movie details into dateadded (asc) # order before outputting the details. # # Since it's possible for movies to be added with the # same time, add an extra component to ensure the key # is unique. unique = 0 for imdb in dupelist: dupes = {} for movie in data: if imdb == movie["imdbnumber"]: unique += 1 dupes["%s.%d" % (movie["dateadded"], unique)] = movie gLogger.out("IMDb Number: %s" % imdb, newLine=True) for dupekey in sorted(dupes): movie = dupes[dupekey] gLogger.out(" Title: %s" % movie["title"], newLine=True) gLogger.out(" Movie ID: %d" % movie["movieid"], newLine=True) gLogger.out(" Playcount: %d" % movie["playcount"], newLine=True) gLogger.out(" Last Played: %s" % movie["lastplayed"], newLine=True) gLogger.out(" Date Added: %s" % movie["dateadded"], newLine=True) gLogger.out(" File: %s" % movie["file"], newLine=True) gLogger.out("", newLine=True) def updateIMDb(mediatype, jcomms, data): if mediatype == "movies": imdbfields = gConfig.IMDB_FIELDS_MOVIES else: imdbfields = gConfig.IMDB_FIELDS_TVSHOWS plotFull = ("plot" in imdbfields) plotOutline = ("plotoutline" in imdbfields) movies250 = None if "top250" in imdbfields: movies250 = MyUtility.Top250MovieList() if movies250 is None: gLogger.err("WARNING: failed to obtain Top250 movies, check log for details", newLine=True) # Movies and TV shows worklist = _ProcessIMDB(mediatype, jcomms, data, plotFull, plotOutline, movies250, imdbfields) # Once we have verified TV shows, add the episodes and process them if mediatype == "tvshows": epsdata = [] tvhash = {} for tvshow in worklist: tvhash[tvshow["libraryid"]] = tvshow for tvshow in data: if tvshow["tvshowid"] not in tvhash: continue if "seasons" not in tvshow: continue multipart = {} for season in tvshow["seasons"]: if season["season"] < 1: continue if "episodes" not in season: continue for episode in season["episodes"]: if "title" not in episode: episode["title"] = episode["label"] episode["imdbnumber"] = tvhash[tvshow["tvshowid"]]["imdbnumber"] SxE = re.sub("[0-9]*x([0-9]*)\..*", "\\1", episode["label"]) if SxE == episode["label"] or int(SxE) < 1: continue file = episode.get("file", None) if file is not None: if file is not None and file in multipart: multipart_first = multipart[file] else: multipart[file] = (season["season"], int(SxE)) multipart_first = None else: multipart_first = None epsdata.append({"ShowTitle": tvshow["title"], "ShowYear": tvshow["year"], "OriginalShowTitle": tvshow["tc.title"], "Season": season["season"], "Episode": int(SxE), "multipart_ep": multipart_first, "item": episode}) # Add episodes requiring updates to the existing list of work items worklist.extend(_ProcessIMDB("episodes", jcomms, epsdata, plotFull, plotOutline, movies250, imdbfields)) # Remove year fields from worklist, as they're not required # Remove anything without items for w in worklist: if "year" in w: del w["year"] # Create a new list without any workitems that have no work to perform (ie. items=={}) newlist = [x for x in worklist if x["items"]] # Sort the list of items into title order, with libraryid to ensure consistency # when two or more movies share the same title. newlist.sort(key=lambda f: (f["title"], f.get("episodetitle", ""), f["libraryid"])) jcomms.dumpJSON(newlist, decode=True, ensure_ascii=True) def _ProcessIMDB(mediatype, jcomms, data, plotFull, plotOutline, movies250, imdbfields): worklist = [] input_queue = Queue.Queue() output_queue = Queue.Queue() # Load input queue # For tvshows, we need to perform some extra processing on the title and year of each show # so that it matches the title/year on OMDb if mediatype == "movies": for movie in data: input_queue.put({"item": movie}) elif mediatype == "tvshows": re_ignore_titles = [] re_map_titles = [] re_trans_titles = [] re_trans_years = [] re_parenthesis = re.compile("\([a-zA-Z]*\)$") for ft in gConfig.IMDB_IGNORE_TVTITLES: re_ignore_titles.append(re.compile(ft, re.IGNORECASE)) for ft in gConfig.IMDB_MAP_TVTITLES: ftsplit = ft.split("=") if len(ftsplit) == 2: re_map_titles.append((re.compile(ftsplit[0], re.IGNORECASE), ftsplit[1])) for ft in gConfig.IMDB_TRANSLATE_TVTITLES: ftsplit = ft.split("=") if len(ftsplit) == 2: re_trans_titles.append((re.compile(ftsplit[0], re.IGNORECASE), ftsplit[1])) for ft in gConfig.IMDB_TRANSLATE_TVYEARS: ftsplit = ft.split("=") if len(ftsplit) == 2 and ftsplit[1]: re_trans_years.append((re.compile(ftsplit[0], re.IGNORECASE), int(ftsplit[1]))) for tvshow in data: ignoreShow = False tvshow["tc.title"] = tvshow["title"] tvshow["tc.year"] = tvshow["year"] for ignore in re_ignore_titles: if ignore.search(tvshow["title"]): gLogger.log("Pre-processing #1 OMDb: Title: [%s] ignoring TV show - matched [%s] in imdb.ignore.tvtitles" % (tvshow["title"], ignore.pattern)) ignoreShow = True break if not ignoreShow: # Map to imdbnumber? for trans in re_map_titles: if trans[0].search(tvshow["title"]): gLogger.log("Pre-processing #2 OMDb: Title: [%s] mapped to imdbnumber for pattern [%s], assigning [%s]" % (tvshow["title"], trans[0].pattern, trans[1])) tvshow["imdbnumber"] = trans[1] break # If we've got an imdbnumber don't bother with remaining translations if "imdbnumber" in tvshow: input_queue.put({"OriginalShowTitle": tvshow["tc.title"], "item": tvshow}) continue # Translate year for trans in re_trans_years: if trans[0].search(tvshow["title"]): gLogger.log("Pre-processing #2 OMDb: Title: [%s] translating TV show year for pattern [%s], replacing [%d] with [%d]" % (tvshow["title"], trans[0].pattern, tvshow["year"], trans[1])) tvshow["year"] = trans[1] # Translate title for trans in re_trans_titles: if trans[0].search(tvshow["title"]): before_title = tvshow["title"] tvshow["title"] = trans[0].sub(trans[1], tvshow["title"]).strip() gLogger.log("Pre-processing #3 OMDb: Title: [%s] translating TV show title for pattern [%s], replacing with [%s], giving [%s]" % (before_title, trans[0].pattern, trans[1], tvshow["title"])) # Remove original year from end of title if tvshow["tc.year"] is not None and tvshow["title"].endswith("(%d)" % tvshow["tc.year"]): before_title = tvshow["title"] tvshow["title"] = re.sub("\(%d\)$" % tvshow["tc.year"], "", tvshow["title"]).strip() gLogger.log("Pre-processing #4 OMDb: Title: [%s] removing year from title, giving [%s]" % (before_title, tvshow["title"])) # Remove trailing parenthesis (...) from end of title - most likely to be a country code if gConfig.IMDB_DEL_PARENTHESIS: re_find = re_parenthesis.search(tvshow["title"]) if re_find: before_title = tvshow["title"] tvshow["title"] = tvshow["title"][:re_find.start() - 1].strip() gLogger.log("Pre-processing #5 OMDb: Title: [%s] removing trailing parenthesis from title, giving [%s]" % (before_title, tvshow["title"])) input_queue.put({"OriginalShowTitle": tvshow["tc.title"], "item": tvshow}) elif mediatype == "episodes": for episode in data: input_queue.put(episode) if input_queue.qsize() == 0: return worklist # Create threads to process input queue threadcount = input_queue.qsize() if input_queue.qsize() <= gConfig.IMDB_THREADS else gConfig.IMDB_THREADS threads = [] for i in range(threadcount): t = MyIMDBLoader(gConfig, gLogger, input_queue, output_queue, plotFull, plotOutline, movies250, imdbfields) threads.append(t) t.setDaemon(True) # Start the threads... for t in threads: t.start() # Process results from output queue # An empty qItem signifies end of thread # Once all threads have ended, exit loop while threadcount > 0: qItem = output_queue.get(block=True) output_queue.task_done() if qItem is None: threadcount -= 1 continue item = qItem["item"] if "OriginalShowTitle" in qItem: title = qItem["OriginalShowTitle"] else: title = item["title"] imdbnumber = item.get("imdbnumber", "") year = item.get("year", None) if "episodeid" in item: infotitle = "%s S%02dE%02d" % (title, qItem["Season"], qItem["Episode"]) else: infotitle = title gLogger.progress("Processing IMDb: %s..." % infotitle) newimdb = qItem["newimdb"] if mediatype == "movies": libid = item["movieid"] elif mediatype == "tvshows": libid = item["tvshowid"] imdbnumber = newimdb.get("id", "") elif mediatype == "episodes": libid = item["episodeid"] imdbnumber = newimdb.get("id", "") # Truncate rating to 1 decimal place # Keep existing value if difference is likely to be due to platform # precision error ie. Python2 on OpenELEC where 6.6 => 6.5999999999999996. if "rating" in imdbfields and newimdb.get("rating",None) is not None: item["rating"] = float("%.1f" % item.get("rating", 0.0)) newimdb["rating"] = float("%.1f" % newimdb.get("rating", 0.0)) if abs(item["rating"] - newimdb["rating"]) < 0.01: newimdb["rating"] = item["rating"] # Sort genre lists for comparison purposes if "genre" in imdbfields and "genre" in newimdb: item["genre"] = sorted(item.get("genre", [])) newimdb["genre"] = sorted(newimdb.get("genre", [])) olditems = {"items": {}} workitem = {"type": mediatype[:-1], "libraryid": libid, "imdbnumber": imdbnumber, "title": title, "items": {}} if year is not None: workitem["year"] = year, if mediatype == "episodes": workitem["episodetitle"] = item["title"] for field in imdbfields: # Don't attempt to change title of tvshows if mediatype == "tvshows" and field in ["title"]: continue # Don't attempt to set genre for episodes - only works at the tvshow level if mediatype == "episodes" and field in ["genre"]: continue # Don't overwrite a movie rating if the movie already has a rating (eg. foreign movie with manually assigned rating) if mediatype == "movies" and field == "mpaa": if item.get(field, None) not in ["Rated Not Rated", "Not Rated", "Rated Unrated", "Unrated", "NR", "null"]: continue if field in newimdb: if field not in item or item[field] != newimdb[field] and newimdb[field] is not None: workitem["items"][field] = newimdb[field] olditems["items"][field] = item.get(field, None) # Always append tvshows - we use this to validate the show when deciding to process episodes. # TVshows without any changes will be dropped at the end. if workitem["items"] or mediatype == "tvshows": worklist.append(workitem) if gLogger.LOGGING: with lock: # Grab threading lock to prevent threads intermingling their output with our main thread gLogger.log("Workitem for id: %d, type: %s, title: %s" % (workitem["libraryid"], workitem["type"], infotitle)) gLogger.log(" Old items: %s" % olditems["items"]) gLogger.log(" New items: %s" % workitem["items"]) gLogger.progress("") return worklist def getIntFloatStr(aField, aValue): # Some fields can only be strings... if aField in ["title", "plot", "plotoutline", "votes", "studio", "description", "artist", "album", "albumartist", "theme", "mood", "style", "comment", "lyrics", "director", "tagline", "originaltitle", "writer", "mpaa", "country", "imdbnumber", "set", "showtitle", "tag", "sorttitle", "premiered", "dateadded", "lastplayed"]: if (aValue.startswith('"') and aValue.endswith('"')) or \ (aValue.startswith("'") and aValue.endswith("'")): return aValue[1:-1] else: return "%s" % aValue # For everything else, try to work out the data type from the value isString = (type(aValue) == str if MyUtility.isPython3 else type(aValue) in [str, unicode]) if isString: if (aValue.startswith('"') and aValue.endswith('"')) or \ (aValue.startswith("'") and aValue.endswith("'")): return aValue[1:-1] if aValue == "null": return None try: if int(aValue) == float(aValue): return int(aValue) except: pass try: return float(aValue) except: return aValue def setDetails_batch(dryRun=True): jcomms = MyJSONComms(gConfig, gLogger) data=[] for line in sys.stdin: data.append(line) gLogger.log("Read %d lines of data from stdin" % len(data)) jdata = json.loads("".join(data)) gLogger.log("Parsed %d items" % len(jdata)) i = 0 for item in jdata: i += 1 kvpairs = [] for key in item["items"]: kvpairs.append(key) kvpairs.append(item["items"][key]) setDetails_worker(jcomms, item["type"], item["libraryid"], kvpairs, item.get("title", None), dryRun, i, len(jdata), typeconversion=False) gLogger.progress("") def setDetails_single(mtype, libraryid, kvpairs, dryRun=True): # Fix Unicode backslash mangling from the command line... ukvpairs = [] for kv in kvpairs: ukvpairs.append(MyUtility.toUnicode(kv).replace("\\n","\n")) jcomms = MyJSONComms(gConfig, gLogger) if not dryRun else None setDetails_worker(jcomms, mtype, libraryid, ukvpairs, None, dryRun, None, None, typeconversion=True) gLogger.progress("") def setDetails_worker(jcomms, mtype, libraryid, kvpairs, title, dryRun, itemnum, maxitems, typeconversion): if mtype == "movie": method = "VideoLibrary.SetMovieDetails" idname = "movieid" elif mtype == "set": if gConfig.JSON_HAS_SETMOVIESET: method = "VideoLibrary.SetMovieSetDetails" idname = "setid" else: gLogger.out("WARNING: %s is not supported by this version of JSON API (%s) - ignored" % (mtype, gConfig.JSON_VER_STR), newLine=True) return elif mtype == "tvshow": method = "VideoLibrary.SetTVShowDetails" idname = "tvshowid" elif mtype == "season": if gConfig.JSON_HAS_SETSEASON: method = "VideoLibrary.SetSeasonDetails" idname = "seasonid" else: gLogger.out("WARNING: %s is not supported by this version of JSON API (%s) - ignored" % (mtype, gConfig.JSON_VER_STR), newLine=True) return elif mtype == "episode": method = "VideoLibrary.SetEpisodeDetails" idname = "episodeid" elif mtype == "musicvideo": method = "VideoLibrary.SetMusicVideoDetails" idname = "musicvideoid" elif mtype == "artist": method = "AudioLibrary.SetArtistDetails" idname = "artistid" elif mtype == "album": method = "AudioLibrary.SetAlbumDetails" idname = "albumid" elif mtype == "song": method = "AudioLibrary.SetSongDetails" idname = "songid" else: gLogger.out("ERROR: %s is not a valid media type for this operation (id: %d)" % (mtype, libraryid), newLine=True) return if libraryid < 1: gLogger.out("ERROR: %d is not a valid libraryid" % libraryid, newLine=True) return mytitle = title if title else "%s %d" % (idname, libraryid) if itemnum: gLogger.progress("Updating %d of %d: %s..." % (itemnum, maxitems, mytitle)) else: gLogger.progress("Updating: %s..." % mytitle) REQUEST = {"method": method, "params": {idname: libraryid}} # Iterate over list of name/value pairs # Build up dictionary of pairs, and with correct data types pairs = {} KEY = "" bKEY = True for pair in kvpairs: if bKEY: KEY = pair else: if pair is None: pairs[KEY] = None elif type(pair) is list: pairs[KEY] = [] for item in pair: if item: pairs[KEY].append(getIntFloatStr(KEY, item) if typeconversion else item) elif (isinstance(pair, basestring)) and pair.startswith("[") and pair.endswith("]"): pairs[KEY] = [] for item in [x.strip() for x in pair[1:-1].split(",")]: if item: pairs[KEY].append(getIntFloatStr(KEY, item) if typeconversion else item) else: pairs[KEY] = getIntFloatStr(KEY, pair) if typeconversion else pair if (pairs[KEY] is None or pairs[KEY] == "") and \ (KEY.startswith("art.") or KEY in ["fanart", "thumbnail", "thumb"]) and \ not gConfig.JSON_HAS_SETNULL: value = "null" if pairs[KEY] is None else "\"%s\"" % pairs[KEY] gLogger.out("WARNING: cannot set null/empty string value on field with " \ "JSON API %s - ignoring %s %6d (%s = %s)" % \ (gConfig.JSON_VER_STR, idname, libraryid, KEY, value), newLine=True, log=True) return bKEY = not bKEY # Iterate over name/value pairs, adding to params property of the request. # Correctly nest parameters using to dot notation. for pair in pairs: R = REQUEST["params"] fc = len(pair.split(".")) i = 0 for field in pair.split("."): i += 1 if not field in R: R[field] = {} if i == fc: if pairs[pair] is None or pairs[pair] == "": R[field] = None else: R[field] = pairs[pair] R = R[field] if dryRun: gLogger.out("### DRY RUN ###", newLine=True) gLogger.out(json.dumps(REQUEST, indent=2, ensure_ascii=True, sort_keys=False), newLine=True) gLogger.out("### DRY RUN ###", newLine=True) else: # Don't bother calling SetFoo if nothing actually being set if len(REQUEST["params"]) > 1: data = jcomms.sendJSON(REQUEST, "libSetDetails") # Extract data, using optional simple search, or complex SQL filter. def sqlExtract(ACTION="NONE", search="", filter="", delete=False, silent=False): database = MyDB(gConfig, gLogger) with database: SQL = [] ESCAPE_CHAR = "`" if not database.usejson else "" ESCAPE = " ESCAPE '%s'" % ESCAPE_CHAR if ESCAPE_CHAR else "" if search != "": SQL.append("WHERE t.url LIKE '%%%s%%' ORDER BY t.cachedurl ASC" % search.replace("'","''")) # Aide-memoire: Why we have to do the following nonsense: http://trac.kodi.tv/ticket/14905 if gConfig.SEARCH_ENCODE: search2 = urllib2.quote(search, "()") if search2 != search: if ESCAPE_CHAR: search2 = search2.replace("%", "%s%%" % ESCAPE_CHAR) SQL.append("WHERE t.url LIKE '%%%s%%'%s ORDER BY t.cachedurl ASC" % (search2, ESCAPE)) elif filter != "": SQL.append("%s " % filter) FSIZE = 0 FCOUNT = 0 ROWS = [] gLogger.progress("Loading database items...") dbrows = [] if SQL: for sql in SQL: rows = database.getRows(filter=sql, allfields=True) gLogger.log("EXECUTED SQL: queried %d rows" % len(rows)) dbrows.extend(rows) else: rows = database.getRows(allfields=True) gLogger.log("EXECUTED SQL: queried %d rows" % len(rows)) dbrows.extend(rows) rpcnt = 100.0 if len(dbrows) != 0: rpcnt = rpcnt / len(dbrows) i = 0 for row in dbrows: if ACTION == "NONE": ROWS.append(row) else: i += 1 gLogger.progress("Parsing [%s] %2.0f%%..." % (row["cachedurl"], rpcnt * i), every = 50) if ACTION == "EXISTS": if not os.path.exists(gConfig.getFilePath(row["cachedurl"])): ROWS.append(row) elif os.path.getsize(gConfig.getFilePath(row["cachedurl"])) == 0: ROWS.append(row) elif ACTION == "STATS": if os.path.exists(gConfig.getFilePath(row["cachedurl"])): FSIZE += os.path.getsize(gConfig.getFilePath(row["cachedurl"])) ROWS.append(row) gLogger.progress("") FCOUNT=len(ROWS) if delete: i = 0 for row in ROWS: i += 1 gLogger.progress("Deleting row %d (%d of %d)..." % (row["textureid"], i, FCOUNT)) database.deleteItem(row["textureid"], row["cachedurl"], warnmissing=False) gLogger.progress("") elif not silent: for row in ROWS: database.dumpRow(row) if ACTION == "STATS": gLogger.out("\nFile Summary: %s files; Total size: %s KB\n\n" % (format(FCOUNT, ",d"), format(int(FSIZE/1024), ",d"))) if (search != "" or filter != "") and not delete and not silent: gLogger.progress("Matching row ids: %s\n" % " ".join("%d" % r["textureid"] for r in ROWS)) # Delete row by id, and corresponding file item def sqlDelete(ids=[]): database = MyDB(gConfig, gLogger) with database: for id in ids: try: database.deleteItem(int(id)) except ValueError: gLogger.out("id %s is not valid\n" % id) continue def orphanCheck(removeOrphans=False): database = MyDB(gConfig, gLogger) dbfiles = {} ddsmap = {} orphanedfiles = [] gLogger.progress("Loading texture cache...") with database: for r in database.getRows(allfields=False): hash = r["cachedurl"] dbfiles[hash] = r ddsmap[os.path.splitext(hash)[0]] = hash gLogger.log("Loaded %d rows from texture cache" % len(dbfiles)) gLogger.progress("Scanning Thumbnails directory...") path = gConfig.getFilePath() for (root, dirs, files) in os.walk(path): newroot = root.replace(path,"") basedir = os.path.basename(newroot) for file in files: if basedir == "": hash = file else: hash = "%s/%s" % (basedir, file) gLogger.progress("Scanning Thumbnails directory [%s]..." % hash, every=25) hash_parts = os.path.splitext(hash) # If it's a DDS file, it should be associated with another # file with the same hash, but different extension. Find # this other file in the ddsmap - if it's there, ignore # the DDS file, otherwise leave the DDS file to be reported # as an orphaned file. if hash_parts[1] == ".dds" and ddsmap.get(hash_parts[0], None): continue row = dbfiles.get(hash, None) if not row: filename = os.path.join(newroot, file) gLogger.log("Orphan file detected: [%s] with likely hash [%s]" % (filename, hash)) orphanedfiles.append(filename) gLogger.progress("") gLogger.log("Identified %d orphaned files" % len(orphanedfiles)) if removeOrphans and gConfig.ORPHAN_LIMIT_CHECK and len(orphanedfiles) > (len(dbfiles)/20): gLogger.log("Something is wrong here, that's far too many orphaned files - 5% limit exceeded!") gLogger.out("Found %s orphaned files for %s database files.\n\n" % (format(len(orphanedfiles), ",d"), format(len(dbfiles), ",d"))) gLogger.out("This is far too many orphaned files for this number of\n") gLogger.out("database files - more than 5% - and something may be wrong.\n\n") gLogger.out("Check your configuration, database, and Thumbnails folder.\n\n") gLogger.out("Add \"orphan.limit.check = no\" to the properties file\n") gLogger.out("if you wish to disable this check.\n") return FSIZE=0 for ofile in orphanedfiles: fsize = os.path.getsize(gConfig.getFilePath(ofile)) FSIZE += fsize gLogger.out("Orphaned file found: name [%s], created [%s], size [%s]%s\n" % \ (ofile, time.ctime(os.path.getctime(gConfig.getFilePath(ofile))), format(fsize, ",d"), ", REMOVING..." if removeOrphans else "")) if removeOrphans: gLogger.log("Removing orphan file: %s" % gConfig.getFilePath(ofile)) os.remove(gConfig.getFilePath(ofile)) gLogger.out("\nSummary: %s files; Total size: %s KB\n\n" \ % (format(len(orphanedfiles),",d"), format(int(FSIZE/1024), ",d"))) def pruneCache(remove_nonlibrary_artwork=False): localfiles = [] (libraryFiles, mediaFiles) = getAllFiles(keyFunction=getKeyFromFilename) re_search = [] # addons re_search.append(re.compile(r"^.*[/\\]\.kodi[/\\]addons[/\\].*")) re_search.append(re.compile(r"^.*[/\\]\.xbmc[/\\]addons[/\\].*")) # mirrors re_search.append(re.compile(r"^http://mirrors.kodi.tv/addons/.*")) re_search.append(re.compile(r"^http://mirrors.xbmc.org/addons/.*")) database = MyDB(gConfig, gLogger) if gConfig.CHUNKED: pruneCache_chunked(database, libraryFiles, mediaFiles, localfiles, re_search) else: pruneCache_fast(database, libraryFiles, mediaFiles, localfiles, re_search) # Prune, with optional remove... if localfiles != []: if remove_nonlibrary_artwork: gLogger.out("Pruning cached images from texture cache...", newLine=True) else: gLogger.out("The following items are present in the texture cache but not the media library:", newLine=True) gLogger.out("", newLine=True) FSIZE = 0 GOTSIZE = gConfig.HAS_THUMBNAILS_FS localfiles.sort(key=lambda row: row["url"]) with database: for row in localfiles: database.dumpRow(row) if GOTSIZE and os.path.exists(gConfig.getFilePath(row["cachedurl"])): FSIZE += os.path.getsize(gConfig.getFilePath(row["cachedurl"])) if remove_nonlibrary_artwork: database.deleteItem(row["textureid"], row["cachedurl"], warnmissing=False) if GOTSIZE: gLogger.out("\nSummary: %s files; Total size: %s KB\n\n" \ % (format(len(localfiles), ",d"), format(int(FSIZE/1024), ",d"))) else: gLogger.out("\nSummary: %s files\n\n" \ % (format(len(localfiles), ",d"))) def pruneCache_fast(database, libraryFiles, mediaFiles, localfiles, re_search): gLogger.progress("Loading texture cache...") dbfiles = {} with database: for r in database.getRows(allfields=True): dbfiles[r["cachedurl"]] = r totalrows = len(dbfiles) gLogger.log("Loaded %d rows from texture cache" % totalrows) gLogger.progress("Processing texture cache...") for rownum, hash in enumerate(dbfiles): gLogger.progress("Processing texture cache... %d%%" % (100 * rownum / totalrows), every=25) pruneCache_processrow(dbfiles[hash], libraryFiles, mediaFiles, localfiles, re_search) gLogger.progress("") def pruneCache_chunked(database, libraryFiles, mediaFiles, localfiles, re_search): gLogger.progress("Loading Textures DB items...") with database: folders = database.getTextureFolders() for fnum, folder in enumerate(folders): gLogger.progress("Loading Textures DB: chunk %2d of %d..." % (fnum+1, len(folders))) dbfiles = [] dbrows = database.getRows(database.getTextureFolderFilter(folder), allfields=True) i = 0 j = len(dbrows) for dbrow in dbrows: i += 1 gLogger.progress("Processing artwork: chunk %2d of %d (%d%%)" % (fnum+1, len(folders), (100 * i / j)), every=25, finalItem=(i == j)) pruneCache_processrow(dbrow, libraryFiles, mediaFiles, localfiles, re_search) gLogger.progress("") def pruneCache_processrow(row, libraryFiles, mediaFiles, localfiles, re_search): URL = row["url"] isRetained = False if gConfig.PRUNE_RETAIN_TYPES: for retain in gConfig.PRUNE_RETAIN_TYPES: if retain.search(URL): gLogger.log("Retained image due to rule [%s]" % retain.pattern) isRetained = True break if isRetained: return if URL in libraryFiles: del libraryFiles[URL] return if gConfig.PRUNE_RETAIN_CHAPTERS and URL.startswith("chapter://"): if getMediaForChapter(URL) in mediaFiles: return if re_search: # Ignore add-on/mirror related images for r in re_search: if r.search(URL): isRetained = True break # Not an addon or mirror... if not isRetained: localfiles.append(row) def getMediaForChapter(filename): offset = 1 while offset < len(filename): if filename[-offset] in ["/", "\\"]: break offset += 1 return filename[10:-offset] def getHash(string): string = string.lower() bytes = bytearray(string) crc = 0xffffffff; for b in bytes: crc = crc ^ (b << 24) for i in range(8): if (crc & 0x80000000): crc = (crc << 1) ^ 0x04C11DB7 else: crc = crc << 1; crc = crc & 0xFFFFFFFF return "%08x" % crc # The following method is extremely slow on a Raspberry Pi, and # doesn't work well with Unicode strings (returns wrong hash). # Fortunately, using the encoded URL/filename as the key (next # function) is sufficient for our needs and also about twice # as fast on a Pi. def getKeyFromHash(filename): url = MyUtility.normalise(filename, strip=True) hash = getHash(url) return "%s/%s" % (hash[0:1], hash) def getKeyFromFilename(filename): if not filename: return filename return MyUtility.normalise(filename, strip=True) def getAllFiles(keyFunction): jcomms = MyJSONComms(gConfig, gLogger) afiles = {} mfiles = {} UCAST = {} REQUEST = [ {"method":"AudioLibrary.GetAlbums", "params":{"sort": {"order": "ascending", "method": "label"}, "properties":["title", "fanart", "thumbnail"]}}, {"method":"AudioLibrary.GetArtists", "params":{"sort": {"order": "ascending", "method": "artist"}, "properties":["fanart", "thumbnail"], "albumartistsonly": False}}, {"method":"AudioLibrary.GetSongs", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "fanart", "thumbnail"]}}, {"method":"AudioLibrary.GetGenres", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "thumbnail"]}}, {"method":"VideoLibrary.GetMusicVideos", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "thumbnail", "fanart", "art"]}}, {"method":"VideoLibrary.GetMovies", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "cast", "art", "file"]}}, {"method":"VideoLibrary.GetMovieSets", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}}, {"method":"VideoLibrary.GetGenres", "params":{"sort": {"order": "ascending", "method": "title"}, "type": "movie", "properties":["title", "thumbnail"]}}, {"method":"VideoLibrary.GetGenres", "params":{"sort": {"order": "ascending", "method": "title"}, "type": "tvshow", "properties":["title", "thumbnail"]}}, {"method":"VideoLibrary.GetGenres", "params":{"sort": {"order": "ascending", "method": "title"}, "type": "musicvideo", "properties":["title", "thumbnail"]}}, {"method":"Addons.GetAddons", "params":{"properties":["name", "thumbnail", "fanart"]}} ] for r in REQUEST: mediatype = re.sub(".*\.Get(.*)","\\1",r["method"]) if gConfig.CACHE_EXTRA and mediatype == "Movies": jcomms.addProperties(r, "file") gLogger.progress("Loading %s..." % mediatype) data = jcomms.getDataProxy(mediatype, r, uniquecast=UCAST) for items in data.get("result", {}): if items != "limits": if mediatype in ["MovieSets","Addons","Genres"]: interval = 0 else: interval = int(int(data["result"]["limits"]["total"])/10) interval = 50 if interval > 50 else interval title = "" for i in data["result"][items]: title = i.get("title", i.get("artist", i.get("name", None))) gLogger.progress("Loading %s: %s..." % (mediatype, title), every=interval) if "fanart" in i: afiles[keyFunction(i["fanart"])] = "fanart" if "thumbnail" in i: afiles[keyFunction(i["thumbnail"])] = "thumbnail" for a in i.get("art", {}): afiles[keyFunction(i["art"][a])] = a for c in i.get("cast", []): if "thumbnail" in c: afiles[keyFunction(c["thumbnail"])] = "cast.thumb" if mediatype in ["Artists", "Albums", "Movies"]: for file in jcomms.getExtraArt(i): afiles[keyFunction(file["file"])] = file["type"] if "file" in i: mfiles[i["file"]] = "media" if title != "": gLogger.progress("Parsing %s: %s..." % (mediatype, title)) # Free memory used to cache any GetDirectory() information MyUtility.invalidateDirectoryCache(mediatype) gLogger.progress("Loading TV shows...") REQUEST = {"method":"VideoLibrary.GetTVShows", "params": {"sort": {"order": "ascending", "method": "title"}, "properties":["title", "cast", "art"]}} if gConfig.CACHE_EXTRA: jcomms.addProperties(REQUEST, "file") tvdata = jcomms.getDataProxy("tvshows", REQUEST, uniquecast=UCAST) if "result" in tvdata and "tvshows" in tvdata["result"]: for tvshow in tvdata["result"]["tvshows"]: gLogger.progress("Loading TV show: %s..." % tvshow["title"]) tvshowid = tvshow["tvshowid"] for a in tvshow.get("art", {}): afiles[keyFunction(tvshow["art"][a])] = a for c in tvshow.get("cast", []): if "thumbnail" in c: afiles[keyFunction(c["thumbnail"])] = "cast.thumb" for file in jcomms.getExtraArt(tvshow): afiles[keyFunction(file["file"])] = file["type"] REQUEST = {"method":"VideoLibrary.GetSeasons", "params":{"tvshowid": tvshowid, "sort": {"order": "ascending", "method": "season"}, "properties":["season", "art"]}} seasondata = jcomms.getDataProxy("seasons", REQUEST, uniquecast=UCAST) if "seasons" in seasondata["result"]: SEASON_ALL = True for season in seasondata["result"]["seasons"]: seasonid = season["season"] if seasonid < 0: gLogger.err("WARNING: TV show [%s] has invalid season (%d) - ignored" % (tvshow["title"], seasonid), newLine=True) continue gLogger.progress("Loading TV show: %s, season %d..." % (tvshow["title"], seasonid)) for a in season.get("art", {}): if SEASON_ALL and a in ["poster", "tvshow.poster", "tvshow.fanart", "tvshow.banner"]: SEASON_ALL = False (poster_url, fanart_url, banner_url) = jcomms.getSeasonAll(season["art"][a]) if poster_url: afiles[keyFunction(poster_url)] = "poster" if fanart_url: afiles[keyFunction(fanart_url)] = "fanart" if banner_url: afiles[keyFunction(banner_url)] = "banner" afiles[keyFunction(season["art"][a])] = a REQUEST = {"method":"VideoLibrary.GetEpisodes", "params":{"tvshowid": tvshowid, "season": seasonid, "properties":["cast", "art", "file"]}} episodedata = jcomms.getDataProxy("episodes", REQUEST, uniquecast=UCAST) if "episodes" not in episodedata["result"]: continue # ignore seasons without episodes for episode in episodedata["result"]["episodes"]: episodeid = episode["episodeid"] mfiles[episode["file"]] = "media" for a in episode.get("art", {}): afiles[keyFunction(episode["art"][a])] = a for c in episode.get("cast", []): if "thumbnail" in c: afiles[keyFunction(c["thumbnail"])] = "cast.thumb" # Free memory used to cache any GetDirectory() information MyUtility.invalidateDirectoryCache("TVShows") # Pictures gLogger.progress("Loading Pictures...") pictures = jcomms.getPictures(addPreviews=gConfig.PRUNE_RETAIN_PREVIEWS, addPictures=gConfig.PRUNE_RETAIN_PICTURES) for picture in pictures: afiles[keyFunction(picture["thumbnail"])] = "thumbnail" del pictures # Free memory used to cache any GetDirectory() information MyUtility.invalidateDirectoryCache("Pictures") # PVR Channels if gConfig.HAS_PVR: gLogger.progress("Loading PVR channels...") for channelType in ["tv", "radio"]: REQUEST = {"method":"PVR.GetChannelGroups", "params":{"channeltype": channelType}} pvrdata = jcomms.sendJSON(REQUEST, "libPVR", checkResult=False) if "result" in pvrdata: for channelgroup in pvrdata["result"].get("channelgroups", []): REQUEST = {"method":"PVR.GetChannels", "params":{"channelgroupid": channelgroup["channelgroupid"], "properties": ["channeltype", "channel", "thumbnail"]}} channeldata = jcomms.sendJSON(REQUEST, "libPVR", checkResult=False) if "result" in channeldata: for channel in channeldata["result"].get("channels", []): afiles[keyFunction(channel["thumbnail"])] = "pvr.thumb" # Free memory used to cache any GetDirectory() information MyUtility.invalidateDirectoryCache("PVR") return (afiles, mfiles) def removeMedia(mtype, libraryid): MTYPE = {} MTYPE["movie"] = "Movie" MTYPE["musicvideo"] = "MusicVideo" MTYPE["tvshow"] = "TVShow" MTYPE["episode"] = "Episode" if mtype not in MTYPE: gLogger.out("ERROR: %s is not a valid media type for removal - valid types: %s" % (mtype, ", ".join(MTYPE)), newLine=True) return if libraryid < 1: gLogger.out("ERROR: %d is not a valid libraryid for removal" % libraryid, newLine=True) return jcomms = MyJSONComms(gConfig, gLogger) title = jcomms.getTitleForLibraryItem(mtype, libraryid) if title: gLogger.out("Removing %s %d [%s]... " % (mtype, libraryid, title)) jcomms.removeLibraryItem(mtype, libraryid) gLogger.out("Done", newLine=True) else: gLogger.out("ERROR: does not exist - media type [%s] libraryid [%d]" % (mtype, libraryid), newLine=True) # Remove artwork URLs containing specified patterns, with or without lasthaschcheck def purgeArtwork(patterns, hashType="all", dryRun=True): database = MyDB(gConfig, gLogger) SQL = "WHERE" if not gConfig.USEJSONDB or gConfig.JSON_HAS_FILTERNULLVALUE: if hashType == "hashed": SQL = "%s lasthashcheck != '' and" % SQL elif hashType == "unhashed": SQL = "%s lasthashcheck == '' and" % SQL SQL = "%s url like '%%s'" % SQL with database: for pattern in [x for x in patterns if x != ""]: if len(pattern.replace("%", "")) < gConfig.PURGE_MIN_LEN: gLogger.err("Ignoring [%s] as pattern length (excluding wildcards) is less than " \ "%d characters configured by purge.minlen property" % \ (pattern, gConfig.PURGE_MIN_LEN), newLine=True) continue gLogger.progress("Querying database for pattern: %s" % pattern) sqlpattern = pattern if sqlpattern.find("%") == -1: sqlpattern = "%%%s%%" % sqlpattern rows = database.getRows(filter=(SQL % sqlpattern), order="ORDER BY t.id ASC", allfields=True) # Filter out hashed/unhashed rows if JSON API ignores null values on the filter... if gConfig.USEJSONDB and not gConfig.JSON_HAS_FILTERNULLVALUE: newrows = [] for r in rows: if (hashType == "all") or \ (hashType == "hashed" and r["lasthashcheck"]) or \ (hashType == "unhashed" and not r["lasthashcheck"]): newrows.append(r) rows = newrows newrows = None gLogger.out("Purging %d (%s) items for pattern: %s" % (len(rows), hashType, pattern), newLine=True) for r in rows: if dryRun: gLogger.out("Dry-run, would remove: %s" % r["url"], newLine=True) else: gLogger.progress("Removing: %s" % r["url"]) database.deleteItem(r["textureid"], r["cachedurl"], warnmissing=False) gLogger.progress("") def fix_mangled_artwork_urls(): jcomms = MyJSONComms(gConfig, gLogger) files = get_mangled_artwork(jcomms) workitems = {} for f in files: key = "%s[%d]" % (f["idname"], f["id"]) item = workitems.get(key, {}) if item == {}: item["libraryid"] = f["id"] item["type"] = f["type"] item["title"] = f["title"] item["items"] = {} item["items"][f["art"]] = f["fixedurl"] workitems[key] = item worklist = [] for item in workitems: worklist.append(workitems[item]) jcomms.dumpJSON(worklist, decode=True, ensure_ascii=True) def get_mangled_artwork(jcomms): def addItems(item, mediatype, idname): title = item["label"] id = item[idname] if "fanart" in item: allfiles.append({"type": mediatype, "idname": idname, "id": id, "art": "fanart", "url": item["fanart"], "title": title}) if "thumbnail" in i: allfiles.append({"type": mediatype, "idname": idname, "id": id, "art": "thumbnail", "url": item["thumbnail"], "title": title}) if "art" in item: for a in item["art"]: # ignore artwork such as "tvshow.banner" which is a tvshow banner at the episode or season level if a.find(".") == -1: allfiles.append({"type": mediatype, "idname": idname, "id": item[idname], "art": "art.%s" % a, "url": item["art"][a], "title": title}) allfiles = [] idnames = {"Movies": "movieid", "MovieSets": "setid"} types = {"Movies": "movie", "MovieSets": "set"} REQUEST = [ {"method":"VideoLibrary.GetMovies", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}}, {"method":"VideoLibrary.GetMovieSets", "params":{"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}} ] for r in REQUEST: mediatype = re.sub(".*\.Get(.*)","\\1",r["method"]) idname = idnames[mediatype] mtype = types[mediatype] gLogger.progress("Loading: %s..." % mediatype) data = jcomms.sendJSON(r, "libFiles") for items in data.get("result", {}): if items != "limits": if mediatype == "set": interval = 0 else: interval = int(int(data["result"]["limits"]["total"])/10) interval = 50 if interval > 50 else interval title = "" for i in data["result"][items]: title = i.get("title", i.get("artist", i.get("name", None))) gLogger.progress("Parsing %s: %s..." % (mediatype, title), every=interval) addItems(i, mtype, idname) if title != "": gLogger.progress("Parsing %s: %s..." % (mediatype, title)) gLogger.progress("Loading TV shows...") REQUEST = {"method":"VideoLibrary.GetTVShows", "params": {"sort": {"order": "ascending", "method": "title"}, "properties":["title", "art"]}} tvdata = jcomms.sendJSON(REQUEST, "libTV") if "result" in tvdata and "tvshows" in tvdata["result"]: for tvshow in tvdata["result"]["tvshows"]: gLogger.progress("Loading TV show: %s..." % tvshow["title"]) tvshowid = tvshow["tvshowid"] addItems(tvshow, "tvshow", "tvshowid") REQUEST = {"method":"VideoLibrary.GetSeasons", "params":{"tvshowid": tvshowid, "sort": {"order": "ascending", "method": "season"}, "properties":["season", "art"]}} seasondata = jcomms.sendJSON(REQUEST, "libTV") if "seasons" in seasondata["result"]: for season in seasondata["result"]["seasons"]: seasonid = season["season"] if seasonid < 0: gLogger.err("WARNING: TV show [%s] has invalid season (%d) - ignored" % (tvshow["title"], seasonid), newLine=True) continue gLogger.progress("Loading TV show: [%s, season %d]..." % (tvshow["title"], seasonid)) # Can't set items on season unless seasonid is present... if "seasonid" in season: addItems(season, "season", "seasonid") REQUEST = {"method":"VideoLibrary.GetEpisodes", "params":{"tvshowid": tvshowid, "season": seasonid, "properties":["art"]}} episodedata = jcomms.sendJSON(REQUEST, "libTV") for episode in episodedata["result"]["episodes"]: addItems(episode, "episode", "episodeid") files = [] for f in allfiles: original = MyUtility.normalise(f["url"], strip=True) fixed = MyUtility.fixSlashes(original) if original != fixed: f["fixedurl"] = fixed files.append(f) return files def doLibraryScan(media, path): jcomms = MyJSONComms(gConfig, gLogger) scanMethod = "VideoLibrary.Scan" if media == "video" else "AudioLibrary.Scan" jcomms.scanDirectory(scanMethod, path) if media == "video": return jcomms.vUpdateCount else: return jcomms.aUpdateCount def doLibraryClean(media): jcomms = MyJSONComms(gConfig, gLogger) cleanMethod = "VideoLibrary.Clean" if media == "video" else "AudioLibrary.Clean" jcomms.cleanLibrary(cleanMethod) def getDirectory(path, recurse=False): getDirectoryFiles(MyJSONComms(gConfig, gLogger), path, nodirmsg=True, recurse=recurse) def getDirectoryFiles(jcomms, path, nodirmsg=True, recurse=False): data = jcomms.getDirectoryList(path, use_cache=False) # Invalid path if "result" not in data: if nodirmsg: gLogger.out("Directory \"%s\" not found." % path, newLine=True) return # Empty path if "files" not in data["result"]: if nodirmsg: gLogger.out("Directory \"%s\" is empty." % path, newLine=True) return for file in sorted(data["result"]["files"], key=(lambda x: x["file"]), reverse=False): if "file" in file: ftype = file["filetype"] fname = file["file"] if ftype == "directory": FTYPE = "DIR" FNAME = os.path.dirname(fname) else: FTYPE = "FILE" FNAME = fname gLogger.out("%-4s: %s" % (FTYPE, FNAME), newLine=True) if recurse and FTYPE == "DIR": getDirectoryFiles(jcomms, FNAME, nodirmsg=False, recurse=recurse) def showSources(media=None, withLabel=None): jcomms = MyJSONComms(gConfig, gLogger) mlist = [media] if media else ["video", "music", "pictures", "files", "programs"] for m in mlist: for s in jcomms.getSources(m, labelPrefix=True, withLabel=withLabel): gLogger.out("%s: %s" % (m, s), newLine=True) def setPower(state): if state in ["hibernate", "reboot", "shutdown", "suspend", "exit"]: MyJSONComms(gConfig, gLogger).setPower(state) else: gLogger.out("Invalid power state: %s" % state, newLine=True) def execAddon(addon, params, wait=False): REQUEST = {"method":"Addons.ExecuteAddon", "params": {"addonid": addon, "wait": wait}} if params: REQUEST["params"]["params"] = params MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libAddon") def wake_on_lan(): macaddress = gConfig.MAC_ADDRESS.upper() # Check MAC address format and try to normalise to only 12 hex-digits if len(macaddress) == 12 + 5: macaddress = macaddress.replace(macaddress[2], "") # Determine if MAC address consists of only hex digits hex_digits = set("0123456789ABCDEF") validhex = True for char in macaddress: validhex = (char in hex_digits) if not validhex: break # If not 12 digits or not all hex, throw an exception if len(macaddress) != 12 or not validhex: raise ValueError("Incorrect MAC address format [%s]" % gConfig.MAC_ADDRESS) # Format the hex data as 6 bytes of FF, and 16 repetitions of the target MAC address (102 bytes total) data = "".join(["FF" * 6, macaddress * 16]) # Create the broadcast frame by converting each 2-char hex value to a byte frame = bytearray([]) for i in range(0, len(data), 2): frame.append(int(data[i: i + 2], 16)) # Broadcast data to the LAN as a UDP datagram sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) sock.sendto(frame, ("<broadcast>", 7)) sock.close() def showStatus(idleTime=600): jcomms = MyJSONComms(gConfig, gLogger) STATUS = [] if gConfig.JSON_HAS_PROFILE_SUPPORT: STATUS.append("Current Profile: %s" % gConfig.CURRENT_PROFILE["label"]) REQUEST = {"method": "XBMC.GetInfoBooleans", "params": {"booleans": ["System.ScreenSaverActive", "Library.IsScanningMusic", "Library.IsScanningVideo", "System.HasShutdown", "System.CanSuspend"]}} data = jcomms.sendJSON(REQUEST, "libSSaver") if "result" in data: STATUS.append("Scanning Music: %s" % ("Yes" if data["result"].get("Library.IsScanningMusic", False) else "No")) STATUS.append("Scanning Video: %s" % ("Yes" if data["result"].get("Library.IsScanningVideo", False) else "No")) STATUS.append("ScreenSaver Active: %s" % ("Yes" if data["result"].get("System.ScreenSaverActive", False) else "No")) STATUS.append("Suspend Supported: %s" % ("Yes" if data["result"].get("System.CanSuspend", False) else "No")) STATUS.append("Idle Timer Enabled: %s" % ("Yes" if data["result"].get("System.HasShutdown", False) else "No")) property = "System.IdleTime(%s) " % idleTime REQUEST = {"method": "XBMC.GetInfoBooleans", "params": {"booleans": [property]}} data = jcomms.sendJSON(REQUEST, "libIdleTime") if "result" in data: STATUS.append("System Idle > %ss: %s" % (idleTime, ("Yes" if data["result"].get(property, False) else "No"))) STATUS.append("PVR Enabled: %s" % ("Yes" if gConfig.HAS_PVR else "No")) REQUEST = {"method":"Player.GetActivePlayers"} data = jcomms.sendJSON(REQUEST, "libGetPlayers") if "result" in data: if data["result"] == []: STATUS.append("Player: None") else: for player in data["result"]: if "playerid" in player: pType = player["type"] pId = player["playerid"] STATUS.append("Player: %s" % pType.capitalize()) REQUEST = {"method": "Player.GetItem", "params": {"playerid": pId}} data = jcomms.sendJSON(REQUEST, "libGetItem") if "result" in data and "item" in data["result"]: item = data["result"]["item"] iType = item.get("type", None) libraryId = item.get("id", None) if libraryId is None and "label" in item: title = item["label"] elif iType == "song": title = jcomms.getSongName(libraryId) elif iType == "movie": title = jcomms.getMovieName(libraryId) elif iType == "episode": title = jcomms.getEpisodeName(libraryId) elif iType == "musicvideo": title = jcomms.getMusicVideoName(libraryId) else: title = None STATUS.append("Activity: %s" % iType.capitalize()) STATUS.append("Title: %s" % title) REQUEST = {"method": "Player.GetProperties", "params": {"playerid": pId, "properties": ["percentage", "time", "totaltime", "speed"]}} data = jcomms.sendJSON(REQUEST, "libGetProps", checkResult=False) if "result" in data: eTime = getSeconds(data["result"].get("time",0)) tTime = getSeconds(data["result"].get("totaltime",0)) elapsed = getHMS(eTime) pcnt = data["result"].get("percentage", 0) remaining = getHMS(tTime - eTime) if data["result"].get("speed",0) == 0: STATUS.append("Progress: %s (%4.2f%%, %s remaining, paused)" % (elapsed, pcnt, remaining)) elif data["result"].get("speed",0) != 1: STATUS.append("Progress: %s (%4.2f%%, %s remaining, seeking)" % (elapsed, pcnt, remaining)) else: STATUS.append("Progress: %s (%4.2f%%, %s remaining)" % (elapsed, pcnt, remaining)) if STATUS != []: for x in STATUS: pos = x.find(":") gLogger.out("%-20s: %s" % (x[:pos], x[pos+2:]), newLine=True) def getSeconds(aTime): return (aTime["hours"] * 3600) + (aTime["minutes"] * 60) + aTime["seconds"] + (aTime["milliseconds"] / 1000) def getHMS(seconds): return "%02d:%02d:%02d" % (int(seconds/3600) % 24, int(seconds/60) % 60, seconds % 60) def showNotifications(): MyJSONComms(gConfig, gLogger).listen() def rbphdmi(delay): def rbphdmi_listen(self, method, params): cmdqueue.put({"method": method, "params": params}) return (method == "System.OnQuit") RETRIES = gConfig.RPC_RETRY ATTEMPTS = 0 cmdqueue = Queue.Queue() hdmimgr = MyHDMIManager(gConfig, gLogger, cmdqueue, hdmidelay=delay) hdmimgr.setDaemon(True) hdmimgr.start() gLogger.debug("Connecting to Kodi on %s..." % gConfig.KODI_HOST) while True: try: MyJSONComms(gConfig, gLogger).sendJSON({"method": "JSONRPC.Ping"}, "libListen", callback=rbphdmi_listen, checkResult=False) if RETRIES != 0: gLogger.debug("Kodi exited - waiting for restart...") time.sleep(15.0) ATTEMPTS = 0 else: gLogger.debug("Kodi exited") break except socket.error as e: gLogger.debug("Kodi not responding, retries remaining %d" % (RETRIES - ATTEMPTS)) if ATTEMPTS >= RETRIES: raise time.sleep(5.0) ATTEMPTS += 1 def MediaLibraryStats(media_list): jcomms = MyJSONComms(gConfig, gLogger) METHODS = ["AudioLibrary.GetAlbums", "AudioLibrary.GetArtists", "AudioLibrary.GetSongs", "VideoLibrary.GetMovies", "VideoLibrary.GetMovieSets", "VideoLibrary.GetTVShows", "VideoLibrary.GetEpisodes", "VideoLibrary.GetMusicVideos", "Addons.GetAddons"] lmedia_list = [m.lower() for m in media_list] # Add filters for meta-classes if "audio" in lmedia_list: lmedia_list.extend(["albums", "artists", "songs"]) if "video" in lmedia_list: lmedia_list.extend(["movies", "moviesets", "tvshows", "episodes", "musicvideos"]) # Clean up lmedia_list = [m for m in lmedia_list if m not in ["audio", "video"]] if gConfig.JSON_HAS_PROFILE_SUPPORT: gLogger.out("%-11s: %s" % ("Profile", gConfig.CURRENT_PROFILE["label"]), newLine=True) for m in METHODS: media = re.search(".*Get(.*)", m).group(1) if not lmedia_list or media.lower() in lmedia_list: REQUEST = {"method": m, "params": {"limits": {"start": 0, "end": 1}}} if media == "Artists": REQUEST["params"]["albumartistsonly"] = False data = jcomms.sendJSON(REQUEST, "libStats") if "result" in data and "limits" in data["result"]: gLogger.out("%-11s: %d" % (media, data["result"]["limits"]["total"]), newLine=True) def ProcessInput(args): ACTIONS = ["Back", "ContextMenu", "Down", "ExecuteAction", "Home", "Info", "Left", "Right", "Select", "SendText", "ShowOSD", "Up", "Pause"] if not gConfig.JSON_CODEC_INFO_REMOVED: ACTIONS.append("ShowCodec") if gConfig.JSON_PLAYER_PROCESS_INFO: ACTIONS.append("ShowPlayerProcessInfo") actionparam = [] i = 0 while i < len(args): _action = args[i].lower() for action in ACTIONS: if action.lower() == _action: break else: gLogger.err("Invalid action [%s]" % args[i], newLine=True) return if action in ["ExecuteAction", "SendText", "Pause"]: i += 1 if i >= len(args) or not args[i]: gLogger.err("Insufficient arguments for [%s]" % action, newLine=True) return else: param = args[i] else: param = None i += 1 actionparam.append({"action": action, "param": param}) # If first action is not a Pause or noop, insert a noop in case screensaver # is active otherwise client will ignore the first input if len(actionparam) > 0: if not (actionparam[0]["action"] == "Pause" or \ (actionparam[0]["action"] == "ExecuteAction" and actionparam[0]["param"] == "noop")): actionparam.insert(0, {"action": "ExecuteAction", "param": "noop"}) jcomms = MyJSONComms(gConfig, gLogger) for ap in actionparam: if ap["action"] == "Pause": time.sleep(float(ap["param"])) continue REQUEST = {"method": "Input.%s" % ap["action"]} if ap["action"] == "ExecuteAction": REQUEST["params"] = {"action": ap["param"].lower()} if ap["action"] == "SendText": REQUEST["params"] = {"text": ap["param"], "done": True} data = jcomms.sendJSON(REQUEST, "libInput", checkResult=False) if "result" not in data or data["result"] != "OK": gLogger.err("Unexpected error during [%s] with params [%s]" % (REQUEST["method"], REQUEST["params"]), newLine=True) return def StressTest(viewtype, numitems, pause, repeat, cooldown): MOVES = numitems - 1 COMMANDS = "executeaction firstpage pause 1.5 " if viewtype == "thumbnail": TOTAL_ROWS = int(MOVES/gConfig.POSTER_WIDTH) LAST_COLS = MOVES % gConfig.POSTER_WIDTH d="" if TOTAL_ROWS > 0: for i in range(0, TOTAL_ROWS): d = "left" if d == "right" else "right" COMMANDS = "%s%s" % (COMMANDS, st_move_horizontal(d, (gConfig.POSTER_WIDTH - 1), pause)) if i < TOTAL_ROWS or LAST_COLS >= 0: COMMANDS = "%s%s" % (COMMANDS, st_move_down(pause)) if LAST_COLS > 0: d = "left" if d == "right" else "right" COMMANDS = "%s%s" % (COMMANDS, st_move_horizontal(d, LAST_COLS, pause)) elif viewtype in ["listright", "horizontal"]: COMMANDS = "%s%s" % (COMMANDS, st_list_move("right", MOVES, pause)) elif viewtype in ["listdown", "vertical"]: COMMANDS = "%s%s" % (COMMANDS, st_list_move("down", MOVES, pause)) else: gLogger.err("%s is not a valid viewtype for stress-test" % viewtype, newLine=True) sys.exit(2) command_list = COMMANDS.strip().split(" ") for i in range(0, repeat): start_time = time.time() gLogger.out("Loop %4d of %d, %s over %d GUI items with %s second pause..." % (i+1, repeat, viewtype, numitems, pause), padspaces=False) ProcessInput(command_list) gLogger.out(" %d seconds" % (time.time() - start_time), newLine=True) if cooldown > 0: gLogger.out("Cooldown period: %s seconds..." % cooldown, padspaces=False) time.sleep(cooldown) gLogger.out(" complete", newLine=True) def st_move_horizontal(direction, count, pause): cmd = "" for i in range(0, count): cmd = "%s%s pause %s " % (cmd, direction, pause) return cmd def st_move_right(count, pause): return "right pause %s " % pause if count == 1 else st_move_horizontal("right", count, pause) def st_move_left(count, pause): return st_move_horizontal("left", count, pause) def st_move_down(pause): return "down pause %s " % pause def st_list_move(direction, count, pause): cmd = "" if count > 0: for i in range(0, count): cmd = "%s%s" % (cmd, st_move_down(pause) if direction == "down" else st_move_right(1, pause)) return cmd def showVolume(): REQUEST = {"method": "Application.GetProperties", "params": {"properties": ["volume", "muted"]}} data = MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libVolume") if "result" in data: mute = "muted" if data["result"]["muted"] else "unmuted" gLogger.out("%s %d" % (mute, data["result"]["volume"]), newLine=True) def setVolume(volume): if volume == "mute": REQUEST = {"method": "Application.SetMute", "params": {"mute": True}} elif volume == "unmute": REQUEST = {"method": "Application.SetMute", "params": {"mute": False}} else: try: REQUEST = {"method": "Application.SetVolume", "params": {"volume": int(volume)}} except: gLogger.err("ERROR: volume level [%s] is not a valid integer" % volume, newLine=True) return data = MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libVolume", checkResult=False) if "result" not in data: gLogger.err("ERROR: volume change failed - valid values: 0-100, mute and unmute", newLine=True) def readFile(infile, outfile): jcomms = MyJSONComms(gConfig, gLogger) url = jcomms.getDownloadURL(infile) if url: try: PAYLOAD = jcomms.sendWeb("GET", url, "readFile", rawData=True) if outfile == "-": os.write(sys.stdout.fileno(), PAYLOAD) sys.stdout.flush() else: f = open(outfile, "wb") f.write(PAYLOAD) f.flush() f.close() except httplib.HTTPException as e: gLogger.err("ERROR not authorised access to file: %s" % infile, newLine=True) sys.exit(2) except Exception as e: gLogger.err("ERROR while creating output file: %s" % str(e), newLine=True) sys.exit(2) else: gLogger.err("ERROR file does not exist: %s" % infile, newLine=True) sys.exit(2) return def ShowGUINotification(title, message, displaytime, image): REQUEST = {"method": "GUI.ShowNotification", "params": {"title": title, "message": message}} if displaytime: REQUEST["params"]["displaytime"] = displaytime if image: REQUEST["params"]["image"] = image MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libNotification") def setSettingVariable(name, value): REQUEST = {"method": "Settings.SetSettingValue", "params": {"setting":name, "value": value}} MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libSetSetting", checkResult=True) def getSettingVariable(name): REQUEST = {"method": "Settings.GetSettingValue", "params": {"setting": name}} data = MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libGetSetting", checkResult=True) return data["result"]["value"] def WriteSetting(name, rawvalue): try: value = eval(rawvalue) except: value = rawvalue setSettingVariable(name, value) def ReadSetting(name): try: gLogger.out("%s: %s" % (name, getSettingVariable(name)), newLine=True) except: pass def ReadSettings(pattern=None): REQUEST = {"method": "Settings.GetSettings", "params": {"level": "expert"}} data = MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libSettings", checkResult=True) if pattern: newdata = [] for item in data["result"]["settings"]: if item["id"].find(pattern) != -1: newdata.append(item) gLogger.out(json.dumps(newdata, indent=2, ensure_ascii=True, sort_keys=True), newLine=True) else: gLogger.out(json.dumps(data["result"]["settings"], indent=2, ensure_ascii=True, sort_keys=True), newLine=True) def playerPlay(afile, playerid=None, withWait=False): def _playlisten(id, method, params): return True if method == "Player.OnStop" else False REQUEST = {"method": "Player.Open", "params": {"item": {"file": afile}}} if playerid is not None: playercoreid = playerid if playercoreid == "null": playercoreid = None elif playercoreid == "default": if not gConfig.JSON_HAS_OPEN_PLAYERCORE_DEFAULT: gLogger.err("WARNING: 'default' is not supported by current JSON API", newLine=True) playercoreid = None else: try: playercoreid = int(playerid) except: pass REQUEST["params"]["options"] = {"playercoreid": playercoreid} MyJSONComms(gConfig, gLogger).sendJSON(REQUEST, "libPlayer", checkResult=True, callback=_playlisten if withWait else None) def playerStop(playerid=None): doplayeraction("Player.Stop", playerid) def playerPause(playerid=None): doplayeraction("Player.PlayPause", playerid) def doplayeraction(action, playerid=None): jcomms = MyJSONComms(gConfig, gLogger) if playerid is None: REQUEST = {"method": "Player.GetActivePlayers"} data = jcomms.sendJSON(REQUEST, "libPlayers", checkResult=False) players = data.get("result", []) for player in players: REQUEST = {"method": action, "params": {"playerid": player["playerid"]}} jcomms.sendJSON(REQUEST, "libPlayer", checkResult=True) else: REQUEST = {"method": action, "params": {"playerid": playerid}} jcomms.sendJSON(REQUEST, "libPlayer", checkResult=True) #--- def pprint(msg): MAXWIDTH=0 line = "Usage: %s" % os.path.basename(__file__) indent = (" " * len(line)) lines = [] for index, token in enumerate(msg.split("|")): token = token.strip().replace(";", "|") if index > 0 and (len(token) + len(line)) > MAXWIDTH: lines.append(line) line = indent line = "%s %s |" % (line, token) lines.append(line[:-2]) lines.append("%s [@property=value ...]" % indent) print("\n".join(lines)) def usage(EXIT_CODE): print("Version: %s" % gConfig.VERSION) print("") pprint("[s, S] <string> | [x, X, f, F] [sql-filter] | Xd | d <id[id id]>] | \ c [class [filter]] | nc [class [filter]] | lc [class] | lnc [class] | C class filter | \ [j, J, jd, Jd, jr, Jr] class [filter] | qa class [filter] | qax class [filter] | [p, P] | [r, R] | \ imdb movies [filter] | imdb tvshows [filter] | \ purge hashed;unhashed;all pattern [pattern [pattern]] | \ purgetest hashed;unhashed;all pattern [pattern [pattern]] | \ fixurls | \ remove mediatype libraryid | \ watched class backup <filename> [filter] | \ watched class restore <filename> [filter] | \ duplicates | \ set | testset | set class libraryid key1 value 1 [key2 value2...] | \ missing class src-label [src-label]* | ascan [path] |vscan [path] | aclean | vclean | \ sources [media] | sources media [label] | directory path | rdirectory path | readfile infile [outfile ; -] | \ notify title message [displaytime [image]] | \ status [idleTime] | monitor | power <state> | exec [params] | execw [params] | wake | \ rbphdmi [seconds] | stats [class]* | \ input action* [parameter] | screenshot | \ volume [mute;unmute;#] | \ stress-test view-type numitems [pause] [repeat] [cooldown] | \ setsetting name value | getsetting name | getsettings [pattern] | debugon | debugoff | \ play item [playerid] | playw item [playerid] | stop [playerid] | pause [playerid] | \ profiles | \ config | version | update | fupdate") print("") print(" s Search URL column for partial movie or TV show title. Case-insensitive.") print(" S Same as \"s\" (search) but will validate cachedurl file exists, displaying only those that fail validation") print(" x Extract details, using optional SQL filter") print(" X Same as \"x\" (extract) but will validate cachedurl file exists, displaying only those that fail validation") print(" Xd Same as \"x\" (extract) but will DELETE those rows for which no cachedurl file exists") print(" f Same as x, but includes file summary (file count, accumulated file size)") print(" F Same as f, but doesn't include database rows") print(" d Delete rows with matching ids, along with associated cached images") print(" c Re-cache missing artwork. Class can be movies, tags, sets, tvshows, artists, albums or songs.") print(" C Re-cache artwork even when it exists. Class can be movies, tags, sets, tvshows, artists, albums or songs. Class and filter both mandatory unless allow.recacheall=yes.") print(" nc Same as c, but don't actually cache anything (ie. see what is missing). Class can be movies, tags, sets, tvshows, artists, albums or songs.") print(" lc Like c, but only for content added since the modification date of the file specficied in property lastrunfile") print(" lnc Like nc, but only for content added since the modification date of the file specficied in property lastrunfile") print(" lC Like C, but only for content added since the modification date of the file specficied in property lastrunfile") print(" j Query library by class (movies, tags, sets, tvshows, artists, albums or songs) with optional filter, return JSON results.") print(" J Same as \"j\", but includes extra JSON audio/video fields as defined in properties file.") print(" jd, Jd Functionality equivalent to j/J, but all URLs are decoded") print(" jr, Jr Functionality equivalent to j/J, but all URLs are decoded and non-ASCII characters output (ie. \"raw\")") print(" qa Run QA check on movies, tags and tvshows, identifying media with missing artwork or plots") print(" qax Same as qa, but remove and rescan those media items with missing details.") print(" Configure with qa.zero.*, qa.blank.* and qa.art.* properties. Prefix field with ? to render warning only.") print(" p Display files present in texture cache that don't exist in the media library") print(" P Prune (automatically remove) cached items that don't exist in the media library") print(" r Reverse search to identify \"orphaned\" Thumbnail files that are not present in the texture cache database") print(" R Same as \"r\" (reverse search) but automatically deletes \"orphaned\" Thumbnail files") print(" imdb Update IMDb fields (default: ratings and votes) on movies or tvshows - pipe output into set to apply changes to media library. Specify alternate or additional fields with @imdb.fields.movies and @imdb.fields.tvshows") print(" purge Remove cached artwork with URLs containing specified patterns, with or without hash") print(" purgetest Dry-run version of purge") print(" fixurls Output new URLs for movies, sets and TV shows that have URLs containing both forward and backward slashes. Output suitable as stdin for set option") print(" remove Remove a library item - specify type (movie, tvshow, episode or musicvideo) and libraryid") print(" watched Backup or restore movies and tvshows watched status and restore points, to/from the specified text file") print(" duplicates List movies with multiple versions as determined by imdb number") print(" set Set values on objects (movie, tvshow, episode, musicvideo, album, artist, song) eg. \"set movie 312 art.fanart 'http://assets.fanart.tv/fanart/movies/19908/hdmovielogo/zombieland-5145e97ed73a4.png'\"") print(" testset Dry run version of set") print(" missing Locate media files missing from the specified media library, matched against one or more source labels, eg. missing movies \"My Movies\"") print(" ascan Scan entire audio library, or specific path") print(" vscan Scan entire video library, or specific path") print(" aclean Clean audio library") print(" vclean Clean video library") print(" sources List all sources, or sources for specific media type (video, music, pictures, files, programs) or label (eg. \"My Movies\")") print(" directory Retrieve list of files in a specific directory (see sources)") print(" rdirectory Recursive version of directory") print(" readfile Read contents of a remote file, writing output to stdout (\"-\", but not suitable for binary data) or the named file (suitable for binary data)") print(" notify Send notification to Kodi GUI. Requires title and message arguments, with optional displaytime in milliseconds (default 5000) and image/icon location") print(" status Display state of client - ScreenSaverActive, SystemIdle (default 600 seconds), active Player state etc.") print(" monitor Display client event notifications as they occur") print(" power Control power state of client, where state is one of suspend, hibernate, shutdown, reboot and exit") print(" wake Wake (over LAN) the client corresponding to the MAC address specified by property network.mac") print(" exec Execute specified addon, with optional parameters") print(" execw Execute specified addon, with optional parameters and wait (although often wait has no effect)") print(" rbphdmi Manage HDMI power saving on a Raspberry Pi by monitoring Screensaver notifications. Default power-off delay is 900 seconds after screensaver has started.") print(" stats Output media library stats") print(" input Send keyboard/remote control input to client, where action is back, left, right, up, down, executeaction, sendtext etc.") print(" volume Set volume level 0-100, mute or unmute, or display current mute state and volume level") print(" stress-test Stress GUI by walking over library items. View type: thumbnail, horizontal, vertical. Default pause 0.25, repeat 1, cooldown (in seconds) 0.") print(" screenshot Take a screen grab of the current display") print(" setsetting Set the value of the named setting, eg. 'setsetting locale.language English'") print(" getsetting Get the current value of the named setting, eg. 'getsetting locale.language'") print(" getsettings View details of all settings, or those where pattern is contained within id, eg. 'getsettings debug' to view details of all debug-related settings") print(" debugon Enable debugging") print(" debugoff Disable debugging") print(" play Play the specified item (on the specified player: null, default, #)") print(" playw Play the specified item (on the specified player: null, default, #), and wait until playback ends") print(" stop Stop playback of the specified player, or all currently active players") print(" pause Toggle pause/playback of the specified player, or all currently active players") print(" profiles List available profiles") print("") print(" config Show current configuration") print(" version Show current version and check for new version") print(" update Update to new version (if available)") print("") print("Valid media classes: addons, pvr.tv, pvr.radio, artists, albums, songs, movies, sets, tags, tvshows") print("Valid meta classes: audio (artists + albums + songs) and video (movies + sets + tvshows) and all (audio + video + addons + pvr.tv + pvr.radio)") print("Meta classes can be used in place of media classes for: c/C/nc/lc/lnc/j/J/jd/Jd/qa/qax options.") print("") print("SQL Filter fields:") print(" id, cachedurl, height, width, usecount, lastusetime, lasthashcheck, url") sys.exit(EXIT_CODE) # Return the path if not defined (eg. cleared by user) or it exists # Otherwise find it, returning undefined if cannot be found def findexepath(cmd, path): if not path or os.path.exists(path): return path else: try: response = subprocess.check_output(["which", cmd], stderr=subprocess.STDOUT) response = response[:-1] if response.endswith("\n") else response return response except: return None def loadConfig(argv): global DBVERSION, MYWEB, MYSOCKET, MYDB global TOTALS global gConfig, gLogger DBVERSION = MYWEB = MYSOCKET = MYDB = None gConfig = MyConfiguration(argv) gLogger = MyLogger() TOTALS = MyTotals(gConfig.LASTRUNFILE_DATETIME) gLogger.DEBUG = gConfig.DEBUG gLogger.VERBOSE = gConfig.LOGVERBOSE gLogger.OPTION = argv[0] if len(argv) != 0 else "" gLogger.setLogFile(gConfig) gLogger.log("Command line args: %s" % sys.argv) gLogger.log("Current version #: v%s" % gConfig.VERSION) gLogger.log("Current platform : %s" % sys.platform) gLogger.log("Python version #: v%d.%d.%d.%d (%s)" % (sys.version_info[0], sys.version_info[1], \ sys.version_info[2], sys.version_info[4], sys.version_info[3])) def checkConfig(option): jsonNeedVersion = 6 # Web server access optWeb = ["c", "C", "readfile"] # Socket (JSON RPC) access optSocket = ["c", "C", "nc", "lc", "lnc", "lC", "j", "J", "jd", "Jd", "jr", "Jr", "qa","qax","query", "p","P", "remove", "vscan", "ascan", "vclean", "aclean", "directory", "rdirectory", "sources", "status", "monitor", "power", "rbphdmi", "stats", "input", "screenshot", "stress-test", "exec", "execw", "missing", "watched", "duplicates", "set", "testset", "volume", "readfile", "notify", "setsetting", "getsetting", "getsettings", "debugon", "debugoff", "play", "playw", "stop", "pause", "fixurls", "imdb", "profiles"] # Database access (could be SQLite, could be JSON - needs to be determined later) optDb = ["s", "S", "x", "X", "Xd", "f", "F", "c", "C", "nc", "lc", "lnc", "lC", "d", "r", "R", "p", "P", "purge", "purgetest"] # These options require direct filesystem access # Dependency: os.remove(), os.path.exists(), os.path.getsize() optFS1 = ["f", "F", "r", "R", "S", "X", "Xd"] # These options require direct filesystem access unless JSON Texture API is available. # Dependency: os.remove() optFS2 = ["d", "P", "C", "purge"] # Network MAC optMAC = ["wake"] needWeb = (option in optWeb) needSocket = (option in optSocket) needDb = (option in optDb) needFS1 = (option in optFS1) needFS2 = (option in optFS2) needMAC = (option in optMAC) wcomms = None jcomms = None # If we need to work out a value for USEJSONDB, we need to check JSON # to determine the current version of JSON API trySocket = False if needDb or needFS2: if gConfig.DBJSON == "auto": trySocket = True elif gConfig.USEJSONDB: needSocket = True gotWeb = gotSocket = gotDb = gotFS = gotMAC = False jsonGotVersion = 0 if needWeb: try: wcomms = MyJSONComms(gConfig, gLogger, connecttimeout=gConfig.WEB_CONNECTTIMEOUT) REQUEST = {"method": "JSONRPC.Ping"} data = wcomms.sendJSON(REQUEST, "libPing", checkResult=False, useWebServer=True) gotWeb = ("result" in data and data["result"] == "pong") except socket.error: pass if needWeb and not gotWeb: MSG = "FATAL: The task you wish to perform requires that the web server is\n" \ " enabled and running on the Kodi system you wish to connect.\n\n" \ " A connection cannot be established to the following webserver:\n" \ " %s:%s\n\n" \ " Check settings in properties file %s\n" % (gConfig.KODI_HOST, gConfig.WEB_PORT, gConfig.CONFIG_NAME) gLogger.err(MSG) return False if needSocket or trySocket: try: jcomms = MyJSONComms(gConfig, gLogger, connecttimeout=gConfig.RPC_CONNECTTIMEOUT) REQUEST = {"method": "JSONRPC.Version"} data = jcomms.sendJSON(REQUEST, "libVersion", checkResult=False) gotSocket = True if "result" in data and "version" in data["result"]: jsonGotVersion = data["result"]["version"] if type(jsonGotVersion) is dict and "major" in jsonGotVersion: gConfig.SetJSONVersion(jsonGotVersion.get("major",0), jsonGotVersion.get("minor",0), jsonGotVersion.get("patch",0)) jsonGotVersion = jsonGotVersion["major"] if gLogger.VERBOSE and gLogger.LOGGING: gLogger.log("JSON CAPABILITIES: %s" % gConfig.dumpJSONCapabilities()) if gConfig.JSON_HAS_PROFILE_SUPPORT: gConfig.ALL_PROFILES = getallprofiles(jcomms) gConfig.CURRENT_PROFILE = getcurrentprofile(jcomms) gLogger.log("CURRENT PROFILE: %s" % gConfig.CURRENT_PROFILE) if gConfig.PROFILE_ENABLED and gConfig.CURRENT_PROFILE["label"] != gConfig.PROFILE_NAME and option != "profiles": if not switchprofile(jcomms): return False jcomms = MyJSONComms(gConfig, gLogger) REQUEST = {"method": "PVR.GetProperties", "params": {"properties": ["available"]}} data = jcomms.sendJSON(REQUEST, "libPVR", checkResult=False) gConfig.HAS_PVR = ("result" in data and data["result"].get("available", False)) except socket.error: pass if needSocket and not gotSocket: MSG = "FATAL: The task you wish to perform requires that the JSON-RPC server is\n" \ " enabled and running on the Kodi system you wish to connect.\n\n" \ " In addition, ensure that the following options are ENABLED on the\n" \ " Kodi client in Settings -> Services -> Remote control:\n\n" \ " Allow programs on this system to control Kodi\n" \ " Allow programs on other systems to control Kodi\n\n" \ " A connection cannot be established to the following JSON-RPC server:\n" \ " %s:%s\n\n" \ " Check settings in properties file %s\n" % (gConfig.KODI_HOST, gConfig.RPC_PORT, gConfig.CONFIG_NAME) gLogger.err(MSG) return False if needSocket and jsonGotVersion < jsonNeedVersion : MSG = "FATAL: The task you wish to perform requires that a JSON-RPC server with\n" \ " version %d or above of the Kodi JSON-RPC API is provided.\n\n" \ " The JSON-RPC API version of the connected server is: %d (0 means unknown)\n\n" \ " Check settings in properties file %s\n" % (jsonNeedVersion, jsonGotVersion, gConfig.CONFIG_NAME) gLogger.err(MSG) return False # If auto detection enabled, when API level insufficient to read Textures DB # using JSON, fall back to SQLite3 calls if needDb and gConfig.DBJSON == "auto": # Able to use JSON for Textures DB access, no need to check DB availability if gConfig.JSON_HAS_TEXTUREDB: gConfig.USEJSONDB = True gLogger.log("JSON Textures DB API available and will be used to access the Textures DB") else: gConfig.USEJSONDB = False gLogger.log("JSON Textures DB API not supported - will use SQLite to access the Textures DB") # If JSON Textures API is to be used... if gConfig.USEJSONDB: # Don't access Textures database using SQLite # Don't access file system either needDb = needFS2 = False # If DB access required, import SQLite3 module if needDb: global lite try: import sqlite3 as lite try: database = MyDB(gConfig, gLogger) con = database.getDB() if database.DBVERSION < 13: MSG = "WARNING: The SQLite3 database pre-dates Frodo (v12), some problems may be encountered!" gLogger.err(MSG, newLine=True) gLogger.log(MSG) gotDb = True except lite.OperationalError: pass except ImportError: gLogger.log("ERROR: SQLite3 module not imported") lite = None if needDb and not gotDb: MSG = "FATAL: The task you wish to perform requires read/write file\n" \ " access to the Kodi SQLite3 Texture Cache database.\n\n" \ " The following SQLite3 database could not be opened:\n" \ " %s\n\n" \ " Check settings in properties file %s,\n" \ " or upgrade your Kodi client to use a more recent\n" \ " version that supports Textures JSON API.\n" \ % (gConfig.getDBPath(), gConfig.CONFIG_NAME) gLogger.err(MSG) return False if (needFS1 or needFS2) and not gConfig.HAS_THUMBNAILS_FS: MSG = "FATAL: The task you wish to perform requires read/write file\n" \ " access to the Thumbnails folder, which is inaccessible.\n\n" \ " Specify the location of this folder using the thumbnails property\n" \ " as an absolute path or relative to the userdata property.\n\n" \ " The currently configured Thumbnails path is:\n" \ " %s\n\n" \ " Check userdata and thumbnails settings in properties file %s\n" % (gConfig.getFilePath(), gConfig.CONFIG_NAME) gLogger.err(MSG) return False if needMAC: gotMAC = (gConfig.MAC_ADDRESS != "") if needMAC and not gotMAC: MSG = "FATAL: The task you wish to perform requires a valid MAC address\n" \ " specified in the property \"network.mac\".\n\n" \ " Check settings in properties file %s\n" % gConfig.CONFIG_NAME gLogger.err(MSG) return False if option == "rbphdmi" or option == "config": gConfig.BIN_TVSERVICE = findexepath("tvservice", gConfig.BIN_TVSERVICE) gConfig.BIN_VCGENCMD = findexepath("vcgencmd", gConfig.BIN_VCGENCMD) if option == "rbphdmi": if not (gConfig.BIN_TVSERVICE and os.path.exists(gConfig.BIN_TVSERVICE)): MSG = "FATAL: The task you wish to perform requires a valid path specified\n" \ " in the property \"bin.tvservice\".\n\n" \ " The current value [%s]\n" \ " is either not set or cannot be accessed.\n\n" \ " Check settings in properties file %s\n" % (gConfig.BIN_TVSERVICE, gConfig.CONFIG_NAME) gLogger.err(MSG) return False gConfig.postConfig() if gLogger.VERBOSE and gLogger.LOGGING: gLogger.log("CONFIG VALUES: \n%s" % gConfig.dumpMemberVariables()) return True def loadprofile(jcomms): REQUEST = {"method": "Profiles.LoadProfile", "params": {"profile": gConfig.PROFILE_NAME, "prompt": False}} if gConfig.PROFILE_PASSWORD != "": REQUEST["params"]["password"] = {"value": gConfig.PROFILE_PASSWORD} REQUEST["params"]["password"]["encryption"] = "md5" if gConfig.PROFILE_ENCRYPTED else "none" if jcomms is None: jcomms = MyJSONComms(gConfig, gLogger) data = jcomms.sendJSON(REQUEST, "libProfile", checkResult=False, ignoreSocketError=True) if "result" not in data: return False else: return True def getallprofiles(jcomms): REQUEST = {"method": "Profiles.GetProfiles", "params": {"properties": ["thumbnail", "lockmode" ]}} if gConfig.JSON_HAS_PROFILE_DIRECTORY: REQUEST["params"]["properties"].extend(["directory"]) data = jcomms.sendJSON(REQUEST, "libProfile", ignoreSocketError=True) profiles = {} if "result" in data: master = [p for p in data["result"]["profiles"] if p["label"] == gConfig.PROFILE_MASTER] master = None if master == [] else master[0] for profile in data["result"]["profiles"]: setProfileDirectory(master, profile) profiles[profile["label"]] = profile return profiles def getcurrentprofile(jcomms): REQUEST = {"method": "Profiles.GetCurrentProfile", "params": {"properties": ["thumbnail", "lockmode" ]}} if gConfig.JSON_HAS_PROFILE_DIRECTORY: REQUEST["params"]["properties"].extend(["directory"]) data = jcomms.sendJSON(REQUEST, "libProfile", ignoreSocketError=True) if "result" in data: profile = data["result"] master = gConfig.ALL_PROFILES.get(gConfig.PROFILE_MASTER, gConfig.ALL_PROFILES.get("Master user", None)) setProfileDirectory(master, profile) return profile else: return gConfig.CURRENT_PROFILE def setProfileDirectory(master, profile): if gConfig.JSON_HAS_PROFILE_DIRECTORY: master_dir = master["directory"] if master else "special://masterprofile/" if profile["directory"].startswith(master_dir): mdir = MyUtility.PathToHostOS(master_dir) pdir = MyUtility.PathToHostOS(profile["directory"]) profile["tc.profilepath"] = pdir[len(mdir):] else: profile["tc.profilepath"] = profile["directory"] else: profile["directory"] = "" profile["tc.profilepath"] = gConfig.PROFILE_DIRECTORY # Prefix with KODI_BASE if profile path is not an absolute path if os.path.isabs(profile["tc.profilepath"]) == False: profile["tc.profilepath"] = os.path.join(gConfig.KODI_BASE, profile["tc.profilepath"]) def switchprofile(jcomms): if gConfig.CURRENT_PROFILE["label"] == gConfig.PROFILE_NAME: return True elif gConfig.PROFILE_AUTOLOAD: gLogger.log("SWITCHING PROFILE FROM \"%s\" to \"%s\"" % (gConfig.CURRENT_PROFILE["label"], gConfig.PROFILE_NAME)) gLogger.progress("Switching profile from \"%s\" to \"%s\"..." % (gConfig.CURRENT_PROFILE["label"], gConfig.PROFILE_NAME)) if jcomms is None: jcomms = MyJSONComms(gConfig, gLogger) if not loadprofile(jcomms): gLogger.err("ERROR: Profile \"%s\" is not valid!" % gConfig.PROFILE_NAME, newLine=True) return False i = 0 bounce = False while i <= gConfig.PROFILE_RETRY: try: i += 1 time.sleep(1.0) jcomms = MyJSONComms(gConfig, gLogger, connecttimeout=1.0) gConfig.CURRENT_PROFILE = getcurrentprofile(jcomms) if gConfig.CURRENT_PROFILE["label"] == gConfig.PROFILE_NAME: gLogger.log("SWITCHED TO PROFILE: %s" % gConfig.CURRENT_PROFILE) if gConfig.PROFILE_WAIT != 0: gLogger.log("Waiting %d seconds for server to stabilise after loading profile..." % gConfig.PROFILE_WAIT) time.sleep(gConfig.PROFILE_WAIT) break elif bounce: loadprofile(jcomms) bounce = False except Exception as e: bounce = True jcomms = None pass else: gLogger.err("ERROR: Failed to load profile %s" % gConfig.PROFILE_NAME, newLine=True) return False gLogger.progress("") else: gLogger.err("ERROR: Need to switch profiles from \"%s\" to \"%s\", but profile.autoload is not enabled" % (gConfig.CURRENT_PROFILE["label"], gConfig.PROFILE_NAME), newLine=True) return False return True def listProfiles(): print("%s %s %-20s %-50s %s" % ("Active", "Lock", "Name", "Device Path", "Local Path")) for p in gConfig.ALL_PROFILES: profile = gConfig.ALL_PROFILES[p] active = "Yes" if profile["label"] == gConfig.CURRENT_PROFILE["label"] else "No " lockmode = "Yes" if profile["lockmode"] != 0 else "No " print(" %s %s %-20s %-50s %s" % (active, lockmode, profile["label"], profile["directory"], profile["tc.profilepath"])) def checkUpdate(argv, forcedCheck=False): (remoteVersion, remoteHash) = getLatestVersion(argv) if forcedCheck: gLogger.out("Current Version: v%s" % gConfig.VERSION, newLine=True) gLogger.out("Latest Version: %s" % ("v" + remoteVersion if remoteVersion else "Unknown"), newLine=True) gLogger.out("", newLine=True) if remoteVersion and MyUtility.getVersion(remoteVersion) > MyUtility.getVersion(gConfig.VERSION): out_method = gLogger.out if forcedCheck else gLogger.err out_method("A new version of this script is available - use the \"update\" option to apply update.", newLine=True) out_method("", newLine=True) if forcedCheck: url = gConfig.GITHUB.replace("//raw.","//").replace("/master","/blob/master") gLogger.out("Full changelog: %s/CHANGELOG.md" % url, newLine=True) def getLatestVersion(argv): # Need user agent etc. for analytics BITS = "64" if platform.architecture()[0] == "64bit" else "32" ARCH = "ARM" if platform.machine().lower().startswith("arm") else "x86" PLATFORM = platform.system() if PLATFORM.lower() == "darwin": PLATFORM = "Mac OSX" if PLATFORM.lower() == "linux": PLATFORM = "%s %s" % (PLATFORM, ARCH) user_agent = "Mozilla/5.0 (%s; %s_%s; rv:%s) Gecko/20100101 Py-v%d.%d.%d.%d/1.0" % \ (PLATFORM, ARCH, BITS, gConfig.VERSION, sys.version_info[0], sys.version_info[1], sys.version_info[2], sys.version_info[4]) # Construct "referer" to indicate usage: USAGE = "unknown" if argv[0] in ["c", "C", "nc", "lc", "lnc", "lC"]: USAGE = "cache" elif argv[0] in ["j", "J", "jd", "Jd", "jr", "Jr"]: USAGE = "dump" elif argv[0] in ["p", "P"]: USAGE = "prune" elif argv[0] in ["r", "R"]: USAGE = "orphan" elif argv[0] in ["s", "S", "d", "f", "F", "x", "X", "Xd"]: USAGE = "db" elif argv[0] in ["exec", "execw"]: USAGE = "exec" elif argv[0] in ["set", "testset"]: USAGE = "set" elif argv[0] in ["purge", "purgetest"]: USAGE = "purge" elif argv[0] in ["qa", "qax"]: USAGE = "qa" elif argv[0] == "stress-test": USAGE = "stress" elif argv[0] in ["play", "playw", "stop", "pause"]: USAGE = "transport" elif argv[0] in ["query", "missing", "watched", "power", "wake", "status", "monitor", "rbphdmi", "directory", "rdirectory", "sources", "remove", "vscan", "ascan", "vclean", "aclean", "duplicates", "fixurls", "imdb", "stats", "input", "screenshot", "volume", "readfile", "notify", "setsetting", "getsetting", "getsettings", "debugon", "debugoff", "version", "update", "fupdate", "config", "profiles"]: USAGE = argv[0] analytics_url = gConfig.ANALYTICS_GOOD HEADERS = [] HEADERS.append(("User-agent", user_agent)) HEADERS.append(("Referer", "http://www.%s" % USAGE)) remoteVersion = remoteHash = None # Try checking version via Analytics URL (remoteVersion, remoteHash) = getLatestVersion_ex(analytics_url, headers = HEADERS) # If the Analytics call fails, go direct to Github if remoteVersion is None or remoteHash is None: (remoteVersion, remoteHash) = getLatestVersion_ex("%s/%s" % (gConfig.GITHUB, "VERSION")) return (remoteVersion, remoteHash) def getLatestVersion_ex(url, headers=None): GLOBAL_TIMEOUT = socket.getdefaulttimeout() ITEMS = (None, None) try: socket.setdefaulttimeout(5.0) if headers: opener = urllib2.build_opener() opener.addheaders = headers response = opener.open(url) else: response = urllib2.urlopen(url) if MyUtility.isPython3: data = response.read().decode("utf-8") else: data = response.read() items = data.replace("\n","").split(" ") if len(items) == 2: ITEMS = items else: gLogger.log("Bogus data in getLatestVersion_ex(): URL [%s], data [%s]" % (url, data), maxLen=512) except Exception as e: gLogger.log("Exception in getLatestVersion_ex(): URL [%s], text [%s]" % (url, e)) socket.setdefaulttimeout(GLOBAL_TIMEOUT) return ITEMS def downloadLatestVersion(argv, force=False, autoupdate=False): (remoteVersion, remoteHash) = getLatestVersion(argv) if autoupdate and (not remoteVersion or MyUtility.getVersion(remoteVersion) <= MyUtility.getVersion(gConfig.VERSION)): return False if not remoteVersion: gLogger.err("FATAL: Unable to determine version of the latest file, check Internet access and github.com are available.", newLine=True) sys.exit(2) if not force and MyUtility.getVersion(remoteVersion) <= MyUtility.getVersion(gConfig.VERSION): gLogger.err("Current version is already up to date - no update required.", newLine=True) sys.exit(2) try: response = urllib2.urlopen("%s/%s" % (gConfig.GITHUB, "texturecache.py")) data = response.read() except Exception as e: gLogger.log("Exception in downloadLatestVersion(): %s" % e) if autoupdate: return False gLogger.err("FATAL: Unable to download latest file, check Internet access and github.com are available.", newLine=True) sys.exit(2) digest = hashlib.md5() digest.update(data) if (digest.hexdigest() != remoteHash): if autoupdate: return False gLogger.err("FATAL: Checksum of new version is incorrect, possibly corrupt download - abandoning update.", newLine=True) sys.exit(2) path = os.path.realpath(__file__) dir = os.path.dirname(path) if os.path.exists("%s%s.git" % (dir, os.sep)): gLogger.err("FATAL: Might be updating version in Git repository... Abandoning update!", newLine=True) sys.exit(2) try: THISFILE = open(path, "wb") THISFILE.write(data) THISFILE.close() except: if autoupdate: gLogger.err("NOTICE: A new version (v%s) of this script is available." % remoteVersion, newLine=True) gLogger.err("NOTICE: Use the \"--update\" option to apply update.", newLine=True) return False else: gLogger.err("FATAL: Unable to update current file, check you have write access", newLine=True) sys.exit(2) gLogger.err("Successfully updated from v%s to v%s" % (gConfig.VERSION, remoteVersion), newLine=True) return True # # Download new version if available, then replace current # process - os.execl() doesn't return. # # Do nothing if newer version not available. # def autoUpdate(argv): if downloadLatestVersion(argv, force=False, autoupdate=True): args = sys.argv args.append("@checkupdate=no") os.execl(sys.executable, sys.executable, *args) def main(argv): loadConfig(argv) if len(argv) == 0: usage(1) if not checkConfig(argv[0]): sys.exit(2) if gConfig.CHECKUPDATE and argv[0] not in ["version", "update", "fupdate"]: if gConfig.AUTOUPDATE: path = os.path.realpath(__file__) dir = os.path.dirname(path) if os.access(dir, os.W_OK): autoUpdate(argv) else: checkUpdate(argv) EXIT_CODE = 0 multi_call_a = ["albums", "artists", "songs"] multi_call_v = ["movies", "sets", "tvshows"] multi_call = ["addons", "agenres", "vgenres", "pvr.tv", "pvr.radio"] + multi_call_a + multi_call_v if argv[0] == "s" and len(argv) == 2: sqlExtract("NONE", search=argv[1]) elif argv[0] == "S" and len(argv) == 2: sqlExtract("EXISTS", search=argv[1]) elif argv[0] == "x" and len(argv) == 1: sqlExtract("NONE") elif argv[0] == "x" and len(argv) == 2: sqlExtract("NONE", filter=argv[1]) elif argv[0] == "X" and len(argv) == 1: sqlExtract("EXISTS") elif argv[0] == "X" and len(argv) == 2: sqlExtract("EXISTS", filter=argv[1]) elif argv[0] == "Xd" and len(argv) == 1: sqlExtract("EXISTS", delete=True) elif argv[0] == "Xd" and len(argv) == 2: sqlExtract("EXISTS", filter=argv[1], delete=True) elif argv[0] == "f" and len(argv) == 1: sqlExtract("STATS") elif argv[0] == "f" and len(argv) == 2: sqlExtract("STATS", filter=argv[1]) elif argv[0] == "F" and len(argv) == 1: sqlExtract("STATS", silent=True) elif argv[0] == "F" and len(argv) == 2: sqlExtract("STATS", filter=argv[1], silent=True) elif argv[0] in ["c", "C", "nc", "lc", "lnc", "lC", "j", "J", "jd", "Jd", "jr", "Jr", "qa", "qax", "query", "imdb"]: _stats = False if argv[0] in ["j", "J", "jd", "Jd", "jr", "Jr"]: _action = "dump" elif argv[0] in ["qa", "qax"]: _action = "qa" elif argv[0] in ["query", "imdb"]: _action = argv[0] else: _action = "cache" _stats = True _force = True if argv[0] in ["C", "lC"] else False _rescan = True if argv[0] == "qax" else False _lastRun = True if argv[0] in ["lc", "lnc", "lC"] else False _nodownload = True if argv[0] in ["nc", "lnc"] else False _decode = True if argv[0] in ["jd", "Jd", "jr", "Jr"] else False _ensure_ascii=False if argv[0] in ["jr", "Jr"] else True _extraFields= True if argv[0] in ["J", "Jd", "Jr"] else False _filter = "" _query = "" _drop_items = {} if argv[0] != "query": _filter = argv[2] if len(argv) > 2 else "" else: if len(argv) == 3: _query = argv[2] elif len(argv) > 3: _filter = argv[2] _query = argv[3] if _force and not gConfig.RECACHEALL and _filter == "": print("Forcing re-cache of all items is disabled. Enable by setting \"allow.recacheall=yes\" in property file.") sys.exit(2) _multi_call = [] if len(argv) == 1: _multi_call = multi_call if "songs" in _multi_call: _multi_call.remove("songs") else: if argv[1] == "audio": _multi_call = multi_call_a if argv[1] == "video": _multi_call = multi_call_v if argv[1] == "all": _multi_call = multi_call if _action == "imdb" and _multi_call != []: usage(1) if _action == "imdb" and not gConfig.OMDB_API_KEY: gLogger.err("ERROR: imdb functionality is no longer available without a valid API key.", newLine=True) gLogger.err("Visit www.omdbapi.com to sign up for an API key, then add", newLine=True) gLogger.err(" omdb.apikey=<yourkey>", newLine=True) gLogger.err("to texturecache.cfg", newLine=True) sys.exit(2) if _multi_call != [] and not gConfig.HAS_PVR: for item in ["pvr.tv", "pvr.radio"]: if item in _multi_call: _multi_call.remove(item) if _multi_call == [] and len(argv) >= 2: _multi_call.append(argv[1]) if _multi_call != []: for _media in _multi_call: jsonQuery(_action, mediatype=_media, filter=_filter, force=_force, lastRun=_lastRun, nodownload=_nodownload, rescan=_rescan, decode=_decode, ensure_ascii=_ensure_ascii, extraFields=_extraFields, query=_query, drop_items=_drop_items) if _action == "cache": dump_drop_items(_drop_items) if _stats: TOTALS.libraryStats(multi=_multi_call, filter=_filter, lastRun=_lastRun, query=_query) else: usage(1) elif argv[0] == "duplicates": jsonQuery("duplicates", "movies") elif argv[0] == "d" and len(argv) >= 2: sqlDelete(argv[1:]) elif argv[0] == "r": orphanCheck(removeOrphans=False) elif argv[0] == "R": orphanCheck(removeOrphans=True) elif argv[0] == "p" and len(argv) == 1: pruneCache(remove_nonlibrary_artwork=False) elif argv[0] == "P" and len(argv) == 1: pruneCache(remove_nonlibrary_artwork=True) elif argv[0] == "remove" and len(argv) == 3: removeMedia(mtype=argv[1], libraryid=int(argv[2])) elif argv[0] in ["purge", "purgetest"] and len(argv) >= 3: if argv[1] not in ["hashed", "unhashed", "all"]: usage(1) purgeArtwork(argv[2:], hashType=argv[1], dryRun=(argv[0] == "purgetest")) elif argv[0] == "fixurls": fix_mangled_artwork_urls() elif argv[0] == "vscan": EXIT_CODE = doLibraryScan("video", path=argv[1] if len(argv) == 2 else None) elif argv[0] == "ascan": EXIT_CODE = doLibraryScan("audio", path=argv[1] if len(argv) == 2 else None) elif argv[0] == "vclean": doLibraryClean("video") elif argv[0] == "aclean": doLibraryClean("audio") elif argv[0] == "directory" and len(argv) == 2: getDirectory(argv[1], recurse=False) elif argv[0] == "rdirectory" and len(argv) == 2: getDirectory(argv[1], recurse=True) elif argv[0] == "sources" and len(argv) < 3: showSources(media=argv[1] if len(argv) == 2 else None) elif argv[0] == "sources" and len(argv) == 3: showSources(media=argv[1], withLabel=argv[2]) elif argv[0] == "status": if len(argv) == 2: showStatus(idleTime=argv[1]) else: showStatus() elif argv[0] == "monitor": showNotifications() elif argv[0] == "version": checkUpdate(argv, forcedCheck=True) elif argv[0] == "config": gConfig.showConfig() elif argv[0] == "update": downloadLatestVersion(argv, force=False) elif argv[0] == "fupdate": downloadLatestVersion(argv, force=True) elif argv[0] == "power" and len(argv) == 2: setPower(argv[1]) elif argv[0] == "exec" and len(argv) > 1: execAddon(argv[1], argv[2:], wait=False) elif argv[0] == "execw" and len(argv) > 1: execAddon(argv[1], argv[2:], wait=True) elif argv[0] == "missing" and len(argv) >= 3: jsonQuery(action="missing", mediatype=argv[1], labels=argv[2:]) elif argv[0] == "watched" and argv[2] in ["backup", "restore"] and len(argv) in [4, 5]: _filter = "" if len(argv) == 4 else argv[4] if argv[2] == "backup": jsonQuery(action="watched", mediatype=argv[1], filename=argv[3], wlBackup=True, filter=_filter) elif argv[2] == "restore": jsonQuery(action="watched", mediatype=argv[1], filename=argv[3], wlBackup=False, filter=_filter) else: usage(1) elif argv[0] == "wake": wake_on_lan() elif argv[0] in ["set", "testset"] and len(argv) == 1: dryRun = (argv[0] != "set") setDetails_batch(dryRun=dryRun) elif argv[0] in ["set", "testset"] and len(argv) >= 4: dryRun = (argv[0] != "set") mtype = argv[1] libraryid = int(argv[2]) + 0 if len(argv) >= 3 else 0 kvpairs = argv[3:] if len(argv) >= 4 else [] setDetails_single(mtype, libraryid, kvpairs, dryRun=dryRun) elif argv[0] == "rbphdmi": _delay = 900 if len(argv) == 1 else int(argv[1]) rbphdmi(delay=_delay) elif argv[0] == "stats": MediaLibraryStats(argv[1:]) elif argv[0] == "input" and len(argv) > 1: ProcessInput(argv[1:]) elif argv[0] == "screenshot": ProcessInput(["executeaction", "screenshot"]) elif argv[0] == "stress-test" and len(argv) >= 3: viewtype = argv[1] numitems = int(argv[2]) pause = float(argv[3]) if len(argv) > 3 else 0.25 repeat = int(argv[4]) if len(argv) > 4 else 1 cooldown = float(argv[5]) if len(argv) > 5 else 0 StressTest(viewtype, numitems, pause, repeat, cooldown) elif argv[0] == "volume" and len(argv) == 1: showVolume() elif argv[0] == "volume" and len(argv) == 2: setVolume(argv[1]) elif argv[0] == "readfile" and len(argv) >= 2: infile = argv[1] outfile = argv[2] if len(argv) == 3 else "-" readFile(infile, outfile) elif argv[0] == "notify" and len(argv) >= 3: _title = argv[1] _message = argv[2] _displaytime= int(argv[3]) if len(argv) >= 4 else None _image = argv[4] if len(argv) >= 5 else None ShowGUINotification(_title, _message, _displaytime, _image) elif argv[0] == "setsetting" and len(argv) == 3: WriteSetting(argv[1], argv[2]) elif argv[0] == "getsetting" and len(argv) == 2: ReadSetting(argv[1]) elif argv[0] == "getsettings" and len(argv) == 1: ReadSettings() elif argv[0] == "getsettings" and len(argv) == 2: ReadSettings(argv[1]) elif argv[0] == "debugon" and len(argv) == 1: setSettingVariable("debug.showloginfo", True) if gConfig.JSON_HAS_DEBUG_EXTRA_LOG: setSettingVariable("debug.extralogging", True) elif argv[0] == "debugoff" and len(argv) == 1: setSettingVariable("debug.showloginfo", False) if gConfig.JSON_HAS_DEBUG_EXTRA_LOG: setSettingVariable("debug.extralogging", False) elif argv[0] == "play" and len(argv) in [2, 3]: playerPlay(argv[1], argv[2] if len(argv) == 3 else None, False) elif argv[0] == "playw" and len(argv) in [2, 3]: playerPlay(argv[1], argv[2] if len(argv) == 3 else None, True) elif argv[0] == "stop" and len(argv) in [1, 2]: playerStop(int(argv[1]) if len(argv) == 2 else None) elif argv[0] == "pause" and len(argv) in [1, 2]: playerPause(int(argv[1]) if len(argv) == 2 else None) elif argv[0] == "profiles": listProfiles() else: usage(1) MyUtility.logDirectoryCacheStats(totals=True) gLogger.log("Successful completion") sys.exit(EXIT_CODE) if __name__ == "__main__": #https://mail.python.org/pipermail/python-list/2015-October/697689.html - threading bug in Python2 tmp = datetime.datetime.strptime('01-01-1970', '%d-%m-%Y') try: stopped = threading.Event() main(sys.argv[1:]) except (KeyboardInterrupt, SystemExit) as e: if type(e) == SystemExit: sys.exit(int(str(e))) except Exception: if "gLogger" in globals() and gLogger.LOGGING: import logging gLogger.flush() logging.basicConfig(filename=gLogger.LOGFILE.name, level=logging.DEBUG) logging.exception("** Terminating due to unexpected exception **") raise