一 最简单的操作方法
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import requests
def main():
r = requests.get(url, headers=headers, stream=True)
f = open("movie.flv", "wb")
print(r.__dict__) # 查看响应信息
size = 31693740.0
lenth = 0
for chunk in r.iter_content(chunk_size=1024):
lenth += len(chunk)
if chunk:
f.write(chunk)
print(str(int(lenth/size*100))+'%', end='\r')
f.close()
企业微信截图_15905794341677.png
二 多进程下载
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import os
import requests
import progressbar
import multiprocessing
def multi_process_download(download_url, fileName, process_num=1, file_size=0, **kwargs):
'''
数据流多进程写入本地文件
:param process_num:
:param file_size:
:return:
'''
if process_num >= 100:
process_num = 99
bsize = file_size / process_num
plist = []
for i in range(0, process_num):
begin = bsize * i
# 最后一个进程的下载数据大小
if (begin + bsize) > file_size or ((i + 1) == process_num and (begin + bsize) < file_size):
bsize = file_size - begin
p = multiprocessing.Process(target=_process_fetch_file,
args=(i, fileName, begin, bsize, download_url))
plist.append(p)
for p in plist:
p.start()
for p in plist:
p.join(600)
is_timeout = False
for p in plist:
# 结束超时的子进程
if p.is_alive():
is_timeout = True
os.kill(p.pid, signal.SIGKILL)
if is_timeout:
# 下载超时重新下载
return multi_process_download(download_url, fileName, process_num=1, file_size=0, **kwargs)
# 合并文件
download_file_path = kwargs.get('download_file_path')
fp = open(download_file_path, 'wb')
for i in range(0, process_num):
# 分别将每个部分文件按顺序写入新文件
fname = 'part_%02d_%s' % (i + 1, fileName)
print '合并文件 %s 中...' % fname
fpath = download_dir_path + fname
if not os.path.isfile(fpath):
print '-' * 100
continue
fp1 = open(fpath, 'rb')
fp.write(fp1.read())
fp1.close()
try:
# 清理碎片文件
os.remove(fpath)
except:
pass
fp.close()
def _process_fetch_file(self, inx, fileName, begin=0, bsize=0, download_url_=''):
"""
多进程获取文件
"""
end = begin + bsize
__headers = {'Range':'bytes=%s-%s'%(begin, end)}
headers.update(__headers)
r = requests.get(url=download_url_, headers=headers, stream=True, timeout=self.timeout, verify=False)
fname = 'part_%02d_%s' % (inx + 1, fileName)
widgets = ['文件 %s 下载中:' % fname, progressbar.Bar(marker='=', left='[', right=']'), '大小:',
progressbar.FileSize(), ' ,已下载: ', progressbar.Percentage(),
' ,速度:', progressbar.FileTransferSpeed()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=bsize).start()
file_path = download_dir_path + fname
file_size__ = 0
with open(file_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=2048):
if not chunk:
break
f.write(chunk)
f.flush()
file_size__ += len(chunk)
if file_size__ % 20480 == 0:
print 'fname: {0} 已下载字符长度: {1}, 预计长度: {2}'.format(fname, file_size__, bsize)
# pbar.update(file_size__)
pbar.finish()
转播
