1, Download Interface
1. Main interface download display
MainWindow first initializes aria and starts aria2. The startup method is in download.py.
# start aria2
start_aria = StartAria2Thread()
self.threadPool.append(start_aria)
self.threadPool[0].start()
self.threadPool[0].ARIA2RESPONDSIGNAL.connect(self.startAriaMessage)
Then define the add download link interface. There can be multiple addlinkwindows, all of which are stored in self.addlinkwindows'list.
There are three parameters (self, self.callBack, self.persepolis_setting) when calling. Download and add the interface to pass the parameters to the callBack function of the main interface through the callBack function. callBack gets the download information and adds it to the thread pool.
new_download = DownloadLink(gid, self)
self.threadPool.append(new_download)
self.threadPool[len(self.threadPool) - 1].start()
self.threadPool[len(self.threadPool) -
1].ARIA2NOTRESPOND.connect(self.aria2NotRespond)
Note that the addLinkSpiderCallBack function in the main interface is called in the following order:
1. Download add interface to get the information of link changed
2. The download add interface enables the thread AddLinkSpiderThread to try to get the size of the link file and add the thread to the main interface thread pool through parent. And connect the signal of AddLinkSpiderThread to the addLinkSpiderCallBack function of the main thread, and add the pointer child of the download add interface to the parameter of the slot function, so that the main interface can access the download add interface through the child.
self.parent.threadPool[len(self.parent.threadPool) - 1].ADDLINKSPIDERSIGNAL.connect(
partial(self.parent.addLinkSpiderCallBack, child=self))
3. The AddLinkSpiderThread thread sends the result ADDLINKSPIDERSIGNAL signal to the addLinkSpiderCallBack function on the main interface. Note that there are only dict parameters when transmitting and two parameters when connecting.
self.ADDLINKSPIDERSIGNAL.emit(spider_dict)
4. The main interface addLinkSpiderCallBack function calls the download add interface through the child to set the display of file name and size.
This is to add a new thread to the main interface of the download link interface, and then control the sub interface update after the main interface thread finishes executing. Why not open a thread to get the file size in the download link add interface, and then change the download link interface according to the result?
mainwindow.py:
class DownloadLink(QThread):
ARIA2NOTRESPOND = pyqtSignal()
def __init__(self, gid, parent):
QThread.__init__(self)
self.gid = gid
self.parent = parent
def run(self):
# add gid of download to the active gids in temp_db
# or update data base , if it was existed before
try:
self.parent.temp_db.insertInSingleTable(self.gid)
except:
# release lock
self.parent.temp_db.lock = False
dictionary = {'gid': self.gid, 'status': 'active'}
self.parent.temp_db.updateSingleTable(dictionary)
# if request is not successful then persepolis is checking rpc
# connection with download.aria2Version() function
answer = download.downloadAria(self.gid, self.parent)
if answer == False:
version_answer = download.aria2Version()
if version_answer == 'did not respond':
self.ARIA2NOTRESPOND.emit()
class MainWindow(MainWindow_Ui):
def __init__(self, start_in_tray, persepolis_main, persepolis_setting):
super().__init__(persepolis_setting)
self.persepolis_setting = persepolis_setting
self.persepolis_main = persepolis_main
# list of threads
self.threadPool = []
# start aria2
start_aria = StartAria2Thread()
self.threadPool.append(start_aria)
self.threadPool[0].start()
self.threadPool[0].ARIA2RESPONDSIGNAL.connect(self.startAriaMessage)
def addLinkButtonPressed(self, button=None):
addlinkwindow = AddLinkWindow(self, self.callBack, self.persepolis_setting)
self.addlinkwindows_list.append(addlinkwindow)
self.addlinkwindows_list[len(self.addlinkwindows_list) - 1].show()
# callback of AddLinkWindow
def callBack(self, add_link_dictionary, download_later, category):
# write information in data_base
self.persepolis_db.insertInDownloadTable([dict])
self.persepolis_db.insertInAddLinkTable([add_link_dictionary])
# if user didn't press download_later_pushButton in add_link window
# then create new qthread for new download!
if not(download_later):
new_download = DownloadLink(gid, self)
self.threadPool.append(new_download)
self.threadPool[len(self.threadPool) - 1].start()
self.threadPool[len(self.threadPool) -
1].ARIA2NOTRESPOND.connect(self.aria2NotRespond)
# open progress window for download.
self.progressBarOpen(gid)
# notify user
# check that download scheduled or not
if not(add_link_dictionary['start_time']):
message = QCoreApplication.translate("mainwindow_src_ui_tr", "Download Starts")
else:
new_spider = SpiderThread(add_link_dictionary, self)
self.threadPool.append(new_spider)
self.threadPool[len(self.threadPool) - 1].start()
self.threadPool[len(self.threadPool) - 1].SPIDERSIGNAL.connect(self.spiderUpdate)
message = QCoreApplication.translate("mainwindow_src_ui_tr", "Download Scheduled")
notifySend(message, '', 10000, 'no', parent=self)
# see addlink.py file
def addLinkSpiderCallBack(self, spider_dict, child):
# get file_name and file_size
file_name = spider_dict['file_name']
file_size = spider_dict['file_size']
if file_size:
file_size = 'Size: ' + str(file_size)
child.size_label.setText(file_size)
if file_name and not(child.change_name_checkBox.isChecked()):
child.change_name_lineEdit.setText(file_name)
child.change_name_checkBox.setChecked(True)
2. Download add interface
AddLinkWindow, the download add interface, initializes the first parameter self as parent, and then accesses the main interface through this parameter. The second parameter is a callback function, which is used to pass parameters to the main interface. The third parameter passes system settings to the download add interface.
When the download link changes, add AddLinkSpiderThread to the threadPool of the main interface, and connect ADDLINKSPIDERSIGNAL to addLinkSpiderCallBack of the main interface.
new_spider = AddLinkSpiderThread(dict)
self.parent.threadPool.append(new_spider)
self.parent.threadPool[len(self.parent.threadPool) - 1].ADDLINKSPIDERSIGNAL.connect(
partial(self.parent.addLinkSpiderCallBack, child=self))
AddLinkSpiderThread obtains the file size and name information through spider.addLinkSpider and sends it to the addLinkSpiderCallBack function of the main interface. Note that when launching here, there are only dict parameters and two parameters when connecting.
self.ADDLINKSPIDERSIGNAL.emit(spider_dict)
After pressing the OK button, the parameters are passed to the main interface through the call back callback function call.
addlink.py:
class AddLinkSpiderThread(QThread):
ADDLINKSPIDERSIGNAL = pyqtSignal(dict)
def __init__(self, add_link_dictionary):
QThread.__init__(self)
self.add_link_dictionary = add_link_dictionary
def run(self):
try:
# get file name and file size
file_name, file_size = spider.addLinkSpider(self.add_link_dictionary)
spider_dict = {'file_size': file_size, 'file_name': file_name}
# emit results
self.ADDLINKSPIDERSIGNAL.emit(spider_dict)
class AddLinkWindow(AddLinkWindow_Ui):
def __init__(self, parent, callback, persepolis_setting, plugin_add_link_dictionary={}):
super().__init__(persepolis_setting)
self.callback = callback
self.plugin_add_link_dictionary = plugin_add_link_dictionary
self.persepolis_setting = persepolis_setting
self.parent = parent
self.link_lineEdit.textChanged.connect(self.linkLineChanged)
self.ok_pushButton.clicked.connect(partial(
self.okButtonPressed, download_later=False))
self.download_later_pushButton.clicked.connect(
partial(self.okButtonPressed, download_later=True))
# enable when link_lineEdit is not empty and find size of file.
def linkLineChanged(self, lineEdit):
if str(self.link_lineEdit.text()) == '':
self.ok_pushButton.setEnabled(False)
self.download_later_pushButton.setEnabled(False)
else: # find file size
dict = {'link': str(self.link_lineEdit.text())}
# spider is finding file size
new_spider = AddLinkSpiderThread(dict)
self.parent.threadPool.append(new_spider)
self.parent.threadPool[len(self.parent.threadPool) - 1].start()
self.parent.threadPool[len(self.parent.threadPool) - 1].ADDLINKSPIDERSIGNAL.connect(
partial(self.parent.addLinkSpiderCallBack, child=self))
self.ok_pushButton.setEnabled(True)
self.download_later_pushButton.setEnabled(True)
def okButtonPressed(self, button, download_later):
# user submitted information by pressing ok_pushButton, so get information
# from AddLinkWindow and return them to the mainwindow with callback!
# save information in a dictionary(add_link_dictionary).
self.add_link_dictionary = {'referer': referer, 'header': header, 'user_agent': user_agent, 'load_cookies': load_cookies,
'out': out, 'start_time': start_time, 'end_time': end_time, 'link': link, 'ip': ip,
'port': port, 'proxy_user': proxy_user, 'proxy_passwd': proxy_passwd,
'download_user': download_user, 'download_passwd': download_passwd,
'connections': connections, 'limit_value': limit, 'download_path': download_path}
# get category of download
category = str(self.add_queue_comboBox.currentText())
del self.plugin_add_link_dictionary
# return information to mainwindow
self.callback(self.add_link_dictionary, download_later, category)
# close window
self.close()3, summary
1. Parameters passed between threads can be passed through callback functions, signals and slots.
2. Between the master and slave threads, the master thread passes self to the slave thread, and the slave thread can call the functions of the master thread. The slave thread can also pass self to the main thread, which makes function calls to the slave thread
2, Download File
The service to start aria2 is started through subprocess.Popen. The meaning of each option is described in the aria2 interface document.
subprocess Modules allow you to generate new processes, connect their inputs, outputs, error pipes, and get their return codes. This module is intended to replace some old modules and functions os.system, os.popen*, os.spawn
https://docs.python.org/zh-cn/3/library/subprocess.html#subprocess.Popen.communicate
https://blog.csdn.net/qq_34355232/article/details/87709418
subprocess.Popen([aria2d, '--no-conf', '--enable-rpc', '--rpc-listen-port=' + str(port), '--rpc-max-request-size=2M', '--rpc-listen-all', '--quiet=true'], stderr=subprocess.PIPE, stdout=subprocess.PIPE, stdin=subprocess.PIPE, shell=False, creationflags=NO_WINDOW)
Adding a download link is done through an XML-RPC remote call:
server = xmlrpc.client.ServerProxy(server_uri, allow_none=True)
The RPC interface of aria2 is described as follows, which supports JSON-RPC and XML-RPC.
https://aria2.github.io/manual/en/html/aria2c.html#rpc-interface
There are many documents about python's XML-RPC library. Two are found as follows:
https://www.jianshu.com/p/9987913cf734
https://developer.51cto.com/art/201906/597963.htm
GID: aria2 manages each download through the GID index, which is a 64 bit binary number. When RPC is accessed, it is represented as a 16 character hexadecimal string. Generally, aria2 generates the GID for each download link, which can also be specified by the user through the GID option.
Accessing aria2 through XML-RPC
aria2.addUri([secret, ]uris[, options[, position]])¶
To add a download link, URIS is an array of download links, option and position are integers, indicating the location inserted in the download queue, and 0 is the first. If the position parameter is not provided or the position is longer than the queue length, the added download is at the end of the download queue. This method returns the newly registered downloaded GID.
aria2.tellStatus([secret, ]gid[, keys])
This method returns the progress of the specified download GID. Keys is an array of strings that specifies which items need to be queried. If keys are empty or omitted, all items are included. Common projects include GID, status, totalLength, completedLength, downloadSpeed, uploadSpeed, numSeeders, connections, dir, and files.
aria2.tellActive([secret][, keys])
This method queries the status of the active download, and the items queried are similar to aria2.tellStatus.
aria2.removeDownloadResult([secret, ]gid)
Remove the download completed / download error / deleted download from the storage according to GID, and return OK if successful
aria2.remove([secret, ]gid)
Delete the download according to GID. If the download is in progress, stop the download first. The status of the download link changes to removed. Returns the GID of the deletion state.
aria2.pause([secret, ]gid)
Pause the download link of the specified GID, and the status of the download link changes to paused. If the download is active, the download link is placed at the front of the waiting queue. To change the state to waiting, you need to use the aria2.unpause method.
download.py
# get port from persepolis_setting
port = int(persepolis_setting.value('settings/rpc-port'))
# get aria2_path
aria2_path = persepolis_setting.value('settings/aria2_path')
# xml rpc
SERVER_URI_FORMAT = 'http://{}:{:d}/rpc'
server_uri = SERVER_URI_FORMAT.format(host, port)
server = xmlrpc.client.ServerProxy(server_uri, allow_none=True)
# start aria2 with RPC
def startAria():
# in Windows
elif os_type == OS.WINDOWS:
if aria2_path == "" or aria2_path == None or os.path.isfile(str(aria2_path)) == False:
cwd = sys.argv[0]
current_directory = os.path.dirname(cwd)
aria2d = os.path.join(current_directory, "aria2c.exe") # aria2c.exe path
else:
aria2d = aria2_path
# NO_WINDOW option avoids opening additional CMD window in MS Windows.
NO_WINDOW = 0x08000000
if not os.path.exists(aria2d):
logger.sendToLog("Aria2 does not exist in the current path!", "ERROR")
return None
# aria2 command in windows
subprocess.Popen([aria2d, '--no-conf', '--enable-rpc', '--rpc-listen-port=' + str(port),
'--rpc-max-request-size=2M', '--rpc-listen-all', '--quiet=true'],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
shell=False,
creationflags=NO_WINDOW)
time.sleep(2)
# check that starting is successful or not!
answer = aria2Version()
# return result
return answer
# check aria2 release version . Persepolis uses this function to
# check that aria2 RPC connection is available or not.
def aria2Version():
try:
answer = server.aria2.getVersion()
except:
# write ERROR messages in terminal and log
logger.sendToLog("Aria2 didn't respond!", "ERROR")
answer = "did not respond"
return answer
def downloadAria(gid, parent):
# add_link_dictionary is a dictionary that contains user download request
# information.
# get information from data_base
add_link_dictionary = parent.persepolis_db.searchGidInAddLinkTable(gid)
answer = server.aria2.addUri([link], aria_dict)3, Database
Use sqlite3 database tutorial: https://docs.python.org/zh-cn/3/library/sqlite3.html
There are three databases, TempDB, in memory, where real-time data is placed. PluginsDB places new link data from browser plug-ins. Persepolis DB is the main database, which stores download information.
TempDB has two tables. Single DB table stores the GID in download, and queue DB table stores the GID information of download queue.
PersepolisDB has four tables:
Category DB table stores type information, including 'All Downloads',' Single Downloads' and 'Scheduled Downloads'.
Download? DB? Table stores the download status table displayed on the main interface.
Addlink? DB? Table stores the download links added in the download add interface.
Video? Finder? DB? Table stores the information added and downloaded in the download add interface.
# This class manages TempDB
# TempDB contains gid of active downloads in every session.
class TempDB():
def __init__(self):
# temp_db saves in RAM
# temp_db_connection
self.temp_db_connection = sqlite3.connect(':memory:', check_same_thread=False)
def createTables(self):
# lock data base
self.lockCursor()
self.temp_db_cursor.execute("""CREATE TABLE IF NOT EXISTS single_db_table(
self.temp_db_cursor.execute("""CREATE TABLE IF NOT EXISTS queue_db_table(
# persepolis main data base contains downloads information
# This class is managing persepolis.db
class PersepolisDB():
def __init__(self):
# persepolis.db file path
persepolis_db_path = os.path.join(config_folder, 'persepolis.db')
# persepolis_db_connection
self.persepolis_db_connection = sqlite3.connect(persepolis_db_path, check_same_thread=False)
# queues_list contains name of categories and category settings
def createTables(self):
# lock data base
self.lockCursor()
# Create category_db_table and add 'All Downloads' and 'Single Downloads' to it
self.persepolis_db_cursor.execute("""CREATE TABLE IF NOT EXISTS category_db_table(
# download table contains download table download items information
self.persepolis_db_cursor.execute("""CREATE TABLE IF NOT EXISTS download_db_table(
# addlink_db_table contains addlink window download information
self.persepolis_db_cursor.execute("""CREATE TABLE IF NOT EXISTS addlink_db_table(
# video_finder_db_table contains addlink window download information
self.persepolis_db_cursor.execute("""CREATE TABLE IF NOT EXISTS video_finder_db_table(
sqlite3 The module supports two kinds of placeholders: question mark (qmark style) and named placeholder (naming style).
# This is the qmark style:
cur.execute("insert into people values (?, ?)", (who, age))
# And this is the named style:
cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age})
The coalesce function returns the value of the first non empty expression in its parameter, that is, if the parameter is provided, the new parameter will be used; if the new parameter is not provided, the original value will be used.
self.temp_db_cursor.execute("""UPDATE single_db_table SET shutdown = coalesce(:shutdown, shutdown),
status = coalesce(:status, status)
WHERE gid = :gid""", dict)MainWindow creates a CheckDownloadInfoThread during initialization, polls each link in the download, and returns the result to the checkDownloadInfo function in the main interface to update the download status.
# CheckDownloadInfoThread
check_download_info = CheckDownloadInfoThread(self)
self.threadPool.append(check_download_info)
self.threadPool[1].start()
self.threadPool[1].DOWNLOAD_INFO_SIGNAL.connect(self.checkDownloadInfo)
self.threadPool[1].RECONNECTARIASIGNAL.connect(self.reconnectAria)
