aquarium/imageLoader.py at master · muma378/aquarium · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#coding=utf8
'''
Created on 2015.04.09
@author: Administrator
'''

import urllib2
import os
from threading import Lock
from mylog import logger

import configure

_global_lock = Lock()

class imageLoader(object):
    def __init__(self):
        pass

    def down_load_image(self, filepath, filename, url, try_cnt):
        logger.debug("down_load_image [imageLoader]")
        logger.info("get url [imageLoader] : %s" % url)
        if try_cnt <= 0:
            with _global_lock:
                with open(configure.error_image_home, "a+") as f:
                    f.write(url+"\t"+filename+"\n")
            return
        #try:
            #filepath = filepath.decode("utf8").encode("gb2312")
            #filename = filename.decode("utf8").encode("gb2312")
        #except:
            #return
        if not os.path.exists(filepath):
            try:
                os.makedirs(filepath)
            except:
                return
        req = urllib2.Request(url)
        req.add_header('User-agent', configure.user_agent_list[1])
        try:
            response = urllib2.urlopen(req, timeout=configure.timeout)
            res = response.read()
            if len(res) == 0:
                self.down_load_image(filepath, filename, url, try_cnt-1)
                return
            with open(filename, "wb") as jpg:
                jpg.write(res)
            logger.info("download\t"+filepath)
        except Exception as e:
            self.down_load_image(filepath, filename, url, try_cnt-1)
            logger.info("can't get image\t"+filepath)
            logger.error(e)
            return