进程池线程池小练习

一个简单的爬虫基础练习：

from concurrent.futures import ThreadPoolExecutor
import requests, time


def get(url):
    print("GET %s" % url)
    response = requests.get(url)
    time.sleep(2)
    return {"url": url, "content": response.text}


def parse(res):
    res = res.result()
    print("%s parse res is %s" % (res["url"], len(res["content"])))


if __name__ == '__main__':
    urls = [
        "https://www.baidu.com/",
        "https://www.12306.cn/index/",
        "https://www.aliyun.com/"
    ]
    pool = ThreadPoolExecutor(2)
    for url in urls:
        pool.submit(get, url).add_done_callback(parse)

运行结果：

GET https://www.baidu.com/
GET https://www.12306.cn/index/
https://www.12306.cn/index/ parse res is 81417
GET https://www.aliyun.com/
https://www.baidu.com/ parse res is 2443
https://www.aliyun.com/ parse res is 441773

修改一下之前的多线程套接字通信小练习为线程池版：

服务端：

import socket

from concurrent.futures import ThreadPoolExecutor


def communicate(conn):
    while True:
        try:
            data = conn.recv(1024)
            if not data: break
            conn.send(data.upper())

        except ConnectionResetError:
            break
    conn.close()


def server(ip, port):
    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server.bind((ip, port))
    server.listen(5)
    print("server running...")

    while True:
        conn, addr = server.accept()
        pool.submit(communicate, conn)  # 提交到pool
    server.close()


if __name__ == '__main__':
    pool = ThreadPoolExecutor(2)  # 设置一个pool，最多同时允许2个任务同时运行
    server("127.0.0.1", 9090)

客户端：没有变化

import socket

client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(("127.0.0.1", 9090))

while True:
    msg = input(">>:").strip()
    if not msg: continue
    client.send(msg.encode("UTF-8"))
    data = client.recv(1024)
    print(data.decode("UTF-8"))
client.close()

此时运行的结果是：

当启动了3个客户端的时候，只有2个能够同时运行，第3个会处于阻塞状态，因为线程池控制了。