From 7c73585c86743b4fcde608d34ea25b01450279b4 Mon Sep 17 00:00:00 2001 From: kev Date: Mon, 13 Jul 2015 01:34:17 +0800 Subject: [PATCH] update --- youtube/youtube-worker/worker.py | 34 ++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/youtube/youtube-worker/worker.py b/youtube/youtube-worker/worker.py index 56d6ff0..e94de41 100755 --- a/youtube/youtube-worker/worker.py +++ b/youtube/youtube-worker/worker.py @@ -6,27 +6,53 @@ import logging import os import redis +import time import youtube_dl def download(url): - with youtube_dl.YoutubeDL() as ydl: + def hook(info): + + time = int(time.time()) + status = info['status'] + + if status == 'downloading': + rdb.zadd('running', time, url) + elif info['status'] == 'error': + rdb.zrem('running', url) + rdb.zadd('error', time, url) + elif status == 'finished': + rdb.zrem('running', url) + rdb.zadd('finished', time, url) + + if rdb.zrank('finished', url) != None: + return False + + opts = { + 'format': os.getenv('FORMAT', 'best'), + 'progress_hooks': [hook], + } + + with youtube_dl.YoutubeDL(opts) as ydl: ydl.download([url]) + return True + if __name__ == '__main__': logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%FT%T', level='INFO') logging.info('connect redis') - rdb = redis.StrictRedis(host='redis', password=os.getenv('PASSWORD')) + rdb = redis.StrictRedis(host='redis', db=os.getenv('DATABASE', 0), password=os.getenv('PASSWORD')) rdb.ping() while True: try: - _, url = rdb.brpop('urls') + _, url = rdb.brpop('pending') logging.info('process: %s', url) - download(url) + ok = download(url) + logging.info('success: %s', ok) except Exception as ex: logging.error('error: %s', ex)