aiohttp 异步爬虫实战

# 导入 requests 包
import requests,pathlib

# 发送请求
def req(ip):    
    try:
        print(ip)
        r = requests.get(ip,timeout=0.5)        
        if r.status_code == 200:            
            with pathlib.Path("1.txt").open("a") as f:
                f.write(ip+"\n") 
    except:
        pass

for i in range(1,255):
    for j in range(1,255):
        ip="http://" + "192.168."+ str(i) +"."+ str(j)+":8080"        
        req(ip)
print("完成")

import asyncio
import aiohttp,pathlib,time

CONCURRENCY = 500 #linux打开文件最大数1024，windows默认509
semaphore = asyncio.Semaphore(CONCURRENCY)
session = None
async def scrape_api(URL):
   async with semaphore:
       print(URL)
       try:
           async with session.get(URL) as response:
               await asyncio.sleep(1)              
               if response.status == 200:
                   return response
               else:
                   return None  #object int can't be used in 'await' expression
       except:
           pass
        
async def main():
   global session
   timeout = aiohttp.ClientTimeout(total=10)
   session = aiohttp.ClientSession(timeout=timeout)
   scrape_index_tasks = [asyncio.ensure_future(scrape_api(URL)) for URL in ["http://192.168."+ str(i) +"." + str(j) + ":8080" for i in range(1,255) for j in range(1,255)]]
   results = await asyncio.gather(*scrape_index_tasks)
   r2 = [r for r in results if r != None]
   print(*r2,file=open(r'./1.txt','w'))
if __name__ == '__main__':
   time_start=time.time()
   asyncio.get_event_loop().run_until_complete(main())
   time_end=time.time()
   print('耗时：',time_end-time_start)