Python实现多线程下载文件的代码实例

实现简单的多线程下载，需要关注如下几点：
1.文件的大小：可以从reponse header中提取，如“Content-Length:911”表示大小是911字节
2.任务拆分：指定各个线程下载的文件的哪一块，可以通过request header中添加“Range: bytes=300-400”（表示下载300~400byte的内容），注意可以请求的文件的range是[0, size-1]字节的。
3.下载文件的聚合：各个线程将自己下载的文件块保存为临时文件，所有线程都完成后，再将这些临时文件按顺序聚合写入到最终的一个文件中。

实现代码：

#!/usr/bin/python
# -*- coding: utf-8 -*-
# filename: paxel.py
# FROM: http://haodaima.com/code/view/58/full/
# Jay modified it a little and save for further potential usage.'''It is a multi-thread downloading tool It was developed following axel.
 Author: volans
 E-mail: volansw [at] gmail.com
'''import sys
import os
import time
import urllib
from threading import Thread# in case you want to use http_proxy
local_proxies = {'http': 'http://131.139.58.200:8080'}
class AxelPython(Thread, urllib.FancyURLopener):
 '''Multi-thread downloading class. run() is a vitural method of Thread.
 '''
 def __init__(self, threadname, url, filename, ranges=0, proxies={}):
 Thread.__init__(self, name=threadname)
 urllib.FancyURLopener.__init__(self, proxies)
 self.name = threadname
 self.url = url
 self.filename = filename
 self.ranges = ranges
 self.downloaded = 0 def run(self):
 '''vertual function in Thread'''
 try:
 self.downloaded = os.path.getsize(self.filename)
 except OSError:
 #print 'never downloaded'
 self.downloaded = 0 # rebuild start poind
 self.startpoint = self.ranges[0] + self.downloaded # This part is completed
 if self.startpoint >= self.ranges[1]:
 print 'Part %s has been downloaded over.' % self.filename
 return self.oneTimeSize = 16384 # 16kByte/time
 print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1]) self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
 self.urlhandle = self.open(self.url) data = self.urlhandle.read(self.oneTimeSize)
 while data:
 filehandle = open(self.filename, 'ab+')
 filehandle.write(data)
 filehandle.close() self.downloaded += len(data)
 #print "%s" % (self.name)
 #progress = u'\r...' data = self.urlhandle.read(self.oneTimeSize)
def GetUrlFileSize(url, proxies={}):
 urlHandler = urllib.urlopen(url, proxies=proxies)
 headers = urlHandler.info().headers
 length = 0
 for header in headers:
 if header.find('Length') != -1:
 length = header.split(':')[-1].strip()
 length = int(length)
 return length
def SpliteBlocks(totalsize, blocknumber):
 blocksize = totalsize / blocknumber
 ranges = []
 for i in range(0, blocknumber - 1):
 ranges.append((i * blocksize, i * blocksize + blocksize - 1))
 ranges.append((blocksize * (blocknumber - 1), totalsize - 1)) return ranges
def islive(tasks):
 for task in tasks:
 if task.isAlive():
 return True
 return False
def paxel(url, output, blocks=6, proxies=local_proxies):
 ''' paxel
 '''
 size = GetUrlFileSize(url, proxies)
 ranges = SpliteBlocks(size, blocks) threadname = ["thread_%d" % i for i in range(0, blocks)]
 filename = ["tmpfile_%d" % i for i in range(0, blocks)] tasks = []
 for i in range(0, blocks):
 task = AxelPython(threadname[i], url, filename[i], ranges[i])
 task.setDaemon(True)
 task.start()
 tasks.append(task) time.sleep(2)
 while islive(tasks):
 downloaded = sum([task.downloaded for task in tasks])
 process = downloaded / float(size) * 100
 show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
 sys.stdout.write(show)
 sys.stdout.flush()
 time.sleep(0.5) filehandle = open(output, 'wb+')
 for i in filename:
 f = open(i, 'rb')
 filehandle.write(f.read())
 f.close()
 try:
 os.remove(i)
 pass
 except:
 pass filehandle.close()if __name__ == '__main__':
 url = 'http://dldir1.qq.com/qqfile/QQforMac/QQ_V3.1.1.dmg'
 output = 'download.file'
 paxel(url, output, blocks=4, proxies={})

以上就是Python实现多线程下载文件的代码实例。爱到最后，成全你的自由。更多关于Python实现多线程下载文件的代码实例请关注haodaima.com其它相关文章！