IO密集型任务与CPU密集型任务

1 with open('input.txt', 'r') as file:
2     data = file.read()
3 
4 with open('output.txt', 'w') as file:
5     file.write('Hello, World!')

1 import requests
2 
3 response = requests.get('https://www.example.com')
4 print(response.text)

import sqlite3

conn = sqlite3.connect('example.db')
cursor = conn.cursor()

cursor.execute("SELECT * FROM users")
rows = cursor.fetchall()

for row in rows:
    print(row)

conn.close()

from PIL import Image

image = Image.open('image.jpg')

# 图像处理操作...

image.save('processed_image.jpg')

 1 '''
 2 1. 使用异步IO：使用异步编程模型（如异步IO、协程等）可以在等待IO操作的同时执行其他任务，提高程序的并发性和响应能力。
 3    Python提供了asyncio库来支持异步IO操作。
 4 '''
 5 
 6 import asyncio
 7 
 8 
 9 async def fetch_data(url):
10     # 模拟网络请求延迟
11     await asyncio.sleep(1)
12     return f"Data from {url}"
13 
14 
15 async def main():
16     urls = ['http://example.com', 'http://example.org']
17     tasks = []
18 
19     for url in urls:
20         task = asyncio.create_task(fetch_data(url))
21         tasks.append(task)
22 
23     results = await asyncio.gather(*tasks)
24 
25     for result in results:
26         print(result)
27 
28 
29 if __name__ == '__main__':
30     asyncio.run(main())

 1 '''
 2 使用异步IO操作可以提高程序的并发性和响应能力，因为在等待IO操作完成的同时，可以执行其他任务(释放cpu资源，让cpu执行其他线程任务)，而不是被阻塞等待IO操作的结果。
 3 这种并发执行任务的方式可以使得程序更加高效地利用系统资源，从而提升整体的性能。
 4 '''
 5 import asyncio
 6 import aiohttp
 7 
 8 
 9 async def fetch_data(url):
10     async with aiohttp.ClientSession() as session:
11         async with session.get(url) as response:
12             data = await response.text()
13             return f"Data from {url}: {data} \n \n 下一个数据:\n"
14 
15 
16 async def main():
17     urls = ['https://www.baidu.com/', 'http://example.org']
18     tasks = []
19 
20     for url in urls:
21         task = asyncio.create_task(fetch_data(url))
22         tasks.append(task)
23 
24     results = await asyncio.gather(*tasks)
25 
26     for result in results:
27         print(result)
28 
29 
30 if __name__ == '__main__':
31     asyncio.run(main())

 1 '''
 2 2. 使用线程池或进程池：对于一些IO操作，可以使用线程池或进程池来创建多个线程或进程，以便并发地执行IO任务。
 3    这样可以充分利用系统资源，并减少IO操作的等待时间。
 4 '''
 5 
 6 import concurrent.futures
 7 import time
 8 
 9 
10 def fetch_data(url):
11     # 模拟网络请求延迟
12     time.sleep(1)
13     return f"Data from {url}"
14 
15 
16 def main():
17     urls = ['http://example.com', 'http://example.org']
18     results = []
19 
20     with concurrent.futures.ThreadPoolExecutor() as executor:
21         futures = [executor.submit(fetch_data, url) for url in urls]
22 
23         for future in concurrent.futures.as_completed(futures):
24             result = future.result()
25             results.append(result)
26 
27     for result in results:
28         print(result)
29 
30 
31 if __name__ == '__main__':
32     main()

 1 '''
 2 3. 使用缓存和批处理：对于一些重复的IO操作，可以使用缓存来存储已经获取的数据，以减少IO操作次数。
 3    此外，可以将多个小的IO请求合并成一个大的IO请求，以减少IO操作的开销。
 4 '''
 5 
 6 import requests
 7 from functools import lru_cache
 8 
 9 
10 @lru_cache(maxsize=128)
11 def get_data(url):
12     response = requests.get(url)
13     return response.text
14 
15 
16 def main():
17     urls = ['https://www.baidu.com/', 'https://www.baidu.com/']
18     results = []
19 
20     for url in urls:
21         data = get_data(url)
22         results.append(data)
23 
24     for result in results:
25         print(result)
26 
27 
28 if __name__ == '__main__':
29     main()

 1 '''
 2 1. 使用多进程处理cpu密集型任务：由于每个进程都有自己独立的解释器和GIL，可以利用多个CPU核心进行并行执行。
 3    使用multiprocessing模块创建多个进程来执行任务，并通过进程间的通信机制（如队列、管道等）来传递数据
 4 '''
 5 
 6 from multiprocessing import Process
 7 
 8 
 9 def calculate_square(number):
10     result = number * number
11     print(f"Square of {number} is {result}")
12 
13 
14 if __name__ == '__main__':
15     numbers = [1, 2, 3, 4, 5]
16     processes = []
17 
18     for number in numbers:
19         process = Process(target=calculate_square, args=(number,))
20         processes.append(process)
21         process.start()
22 
23     for process in processes:
24         process.join()

 1 '''
 2 2. 使用并行计算库：利用一些专门用于执行并行计算的库，如NumPy、Pandas和Dask等，这些库使用底层的C/C++实现，能够更好地利用CPU资源
 3 '''
 4 import numpy as np
 5 
 6 
 7 def calculate_square(number):
 8     result = number * number
 9     print(f"Square of {number} is {result}")
10 
11 
12 if __name__ == '__main__':
13     numbers = np.array([1, 2, 3, 4, 5])
14     square_results = np.square(numbers)
15 
16     for number, result in zip(numbers, square_results):
17         print(f"Square of {number} is {result}")

 1 '''
 2 3. 使用并发框架：一些并发框架（如concurrent.futures）提供了高级接口，可以方便地进行并发执行。
 3 通过将任务分解为多个小任务，并使用线程池或进程池来执行这些小任务，以充分利用CPU资源。
 4 '''
 5 from concurrent.futures import ThreadPoolExecutor
 6 
 7 
 8 def calculate_square(number):
 9     result = number * number
10     print(f"Square of {number} is {result}")
11 
12 
13 if __name__ == '__main__':
14     numbers = [1, 2, 3, 4, 5]
15 
16     with ThreadPoolExecutor() as executor:
17         futures = [executor.submit(calculate_square, number) for number in numbers]
18 
19         for future in futures:
20             future.result()