开源软件批量下载

import requests
import lxml
import gevent
from lxml import html
import os

from gevent import monkey; monkey.patch_os()
monkey.patch_socket()


from multiprocessing import Process



#view-source:https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/
base_url = "https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/"
url_send="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/"
repodata_url ="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/repodata/"

puppetlab = "http://yum.puppetlabs.com/el/6/products/x86_64/"



def get_rpms(url):

    r = requests.get( url )
    doc = html.fromstring(r.text)
    doc.make_links_absolute( url )
    links = doc.iterlinks()
    rpms = [i[2]  for i in links if i[2].endswith("rpm")]
    return rpms

def  get_gzs(url):
    r = requests.get(url)
    doc = html.fromstring(r.text)
    doc.make_links_absolute(url)
    links = doc.iterlinks()
    rpms = [i[2] for i  in links if i[2].endswith("gz") or i[2].endswith("bz2")]
    return rpms


def download(url, dir):
    r = requests.get( url )
    filename = url.split("/")[-1]
    print(filename)
    f = open( os.path.join(dir, filename) , "wb")
    f.write(r.content)
    f.close()

rpms = get_rpms ( puppetlab )

w = []

for i in rpms:
    print(i)
    g = gevent.spawn(download, i, "puppetlab",)
    w.append(g)

gevent.joinall(w)
gevent.sleep(0.3)