forked from pyload/pyload
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathInfo.py
More file actions
216 lines (155 loc) · 6.93 KB
/
Copy pathInfo.py
File metadata and controls
216 lines (155 loc) · 6.93 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
# -*- coding: utf-8 -*-
# @author: RaNaN
import Queue
import os
import sys
import time
import traceback
from pyload.Api import OnlineStatus
from pyload.Datatype import PyFile
from pyload.Thread.Plugin import PluginThread
class InfoThread(PluginThread):
def __init__(self, manager, data, pid=-1, rid=-1, add=False):
"""Constructor"""
PluginThread.__init__(self, manager)
self.data = data
self.pid = pid #: package id
# [ .. (name, plugin) .. ]
self.rid = rid #: result id
self.add = add #: add packages instead of return result
self.cache = [] #: accumulated data
self.start()
def run(self):
"""run method"""
plugins = {}
container = []
for url, plugintype, pluginname in self.data:
# filter out container plugins
if plugintype == 'container':
container.appen((pluginname, url))
else:
if (plugintype, pluginname) in plugins:
plugins[(plugintype, pluginname)].append(url)
else:
plugins[(plugintype, pluginname)] = [url]
# directly write to database
if self.pid > -1:
for (plugintype, pluginname), urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls, self.updateDB)
self.m.core.files.save()
elif self.add:
for (plugintype, pluginname), urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True)
else:
# generate default result
result = [(url, 0, 3, url) for url in urls]
self.updateCache(pluginname, result)
packs = parseNames([(name, url) for name, x, y, url in self.cache])
self.m.log.debug("Fetched and generated %d packages" % len(packs))
for k, v in packs:
self.m.core.api.addPackage(k, v)
# empty cache
del self.cache[:]
else: #: post the results
for name, url in container:
# attach container content
try:
data = self.decryptContainer(name, url)
except Exception:
traceback.print_exc()
self.m.log.error("Could not decrypt container.")
data = []
for url, plugintype, pluginname in data:
try:
plugins[plugintype][pluginname].append(url)
except Exception:
plugins[plugintype][pluginname] = [url]
self.m.infoResults[self.rid] = {}
for plugintype, pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True)
# force to process cache
if self.cache:
self.updateResult(pluginname, [], True)
else:
# generate default result
result = [(url, 0, 3, url) for url in urls]
self.updateResult(pluginname, result, True)
self.m.infoResults[self.rid]['ALL_INFO_FETCHED'] = {}
self.m.timestamp = time.time() + 5 * 60
def updateDB(self, plugin, result):
self.m.core.files.updateFileInfo(result, self.pid)
def updateResult(self, plugin, result, force=False):
# parse package name and generate result
# accumulate results
self.cache.extend(result)
if len(self.cache) >= 20 or force:
# used for package generating
tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) for name, size, status, url in self.cache]
data = parseNames(tmp)
result = {}
for k, v in data.iteritems():
for url, status in v:
status.packagename = k
result[url] = status
self.m.setInfoResults(self.rid, result)
self.cache = []
def updateCache(self, plugin, result):
self.cache.extend(result)
def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None):
try:
result = [] #: result loaded from cache
process = [] #: urls to process
for url in urls:
if url in self.m.infoCache:
result.append(self.m.infoCache[url])
else:
process.append(url)
if result:
self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname))
cb(pluginname, result)
if process:
self.m.log.debug("Run Info Fetching for %s" % pluginname)
for result in plugin.getInfo(process):
# result = [ .. (name, size, status, url) .. ]
if not type(result) == list:
result = [result]
for res in result:
self.m.infoCache[res[3]] = res # : why don't assign res dict directly?
cb(pluginname, result)
self.m.log.debug("Finished Info Fetching for %s" % pluginname)
except Exception, e:
self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)})
if self.m.core.debug:
traceback.print_exc()
# generate default results
if err:
result = [(url, 0, 3, url) for url in urls]
cb(pluginname, result)
def decryptContainer(self, plugin, url):
data = []
# only works on container plugins
self.m.log.debug("Pre decrypting %s with %s" % (url, plugin))
# dummy pyfile
pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1)
pyfile.initPlugin()
# little plugin lifecycle
try:
pyfile.plugin.setup()
pyfile.plugin.loadToDisk()
pyfile.plugin.decrypt(pyfile)
pyfile.plugin.deleteTmp()
for pack in pyfile.plugin.packages:
pyfile.plugin.urls.extend(pack[1])
data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls)
self.m.log.debug("Got %d links." % len(data))
except Exception, e:
self.m.log.debug("Pre decrypting error: %s" % str(e))
finally:
pyfile.release()
return data