python multithreading and multiprocessing
Multithreading:
Case: scan the surviving hosts in a given network (test by ping, if there is a response, it means the hosts are alive)
Normal version:
#Scan hosts alive in a given network(adopt ping To test,If there is a response, the host is alive)
import sys import subprocess import time def ping(net,start=100,end=200,n=2,w=5): for i in range(start,end+1): ip=net+"."+str(i) command="ping %s -n %d -w %d"%(ip,n,w) print(ip,("through","Impassability")[subprocess.call(command,stdout=open("nul","w"))]) #stdout=open("nul","w") #Do not display the results returned by command execution t1=time.time() if len(sys.argv)!=2: print("Parameter input error!") print("Operation example:") print("test01.py 123.125.114") elif len(sys.argv)==2: net=sys.argv[1] ping(net) t2=time.time() print("Program time consuming%f second!"%(t2-t1)) #195.091611 second
The operation effect is as follows:
In python, there are two ways to create a Thread: one is to use the Thread class to create a Thread
Importing Thread module from Python standard library
from threading import Thread
Create a thread
mthread = threading.Thread(target=function_name, args=(function_parameter1, function_parameterN))
Start the thread just created
mthread .start()
function_name: the name of the method to be executed by the thread
args: the parameter received by the thread execution method. This attribute is a tuple. If there is only one parameter, comma should be added at the end.
Multithreaded version:
import sys import subprocess import time from threading import Thread #stay python There are two ways to create threads. One is to use Thread Class creation # Import Python In the standard library Thread Modular #from threading import Thread # # Create a thread #mthread = threading.Thread(target=function_name, args=(function_parameter1, function_parameterN)) # Start the thread you just created #mthread .start() #function_name: The name of the method to be executed by the thread #args: The parameter received by the thread execution method. This attribute is a tuple. If there is only one parameter, comma should be added at the end. result=[] def ping1(ip): command="ping %s -n 1 -w 20"%(ip) result.append([ip,subprocess.call(command)]) def ping(net,start=100,end=200): for i in range(start,end+1): ip=net+"."+str(i) th=Thread(target=ping1,args=(ip,)) th.start() def main(): if len(sys.argv)!=2: print("Parameter input error!") print("Operation example:") print("test01.py 123.125.114") elif len(sys.argv)==2: net=sys.argv[1] ping(net) if __name__=='__main__': t1=time.time() main() while len(result)!=101: time.sleep(1) print(result) t2=time.time() print("Program time consuming%f second!"%(t2-t1)) #1.585263 second
Multithreading case 2: crawling stock price
Multithreading #Crawling stock prices import requests import re import time from threading import Thread code=[600016,600000,601939,600036,603683,600050,601890,600795,601857,600584,601231,603165,600644,603005,601198,603690,600643,600131,600776,603609,601377] m1=re.compile(r"price: '(\d\.\d')") def getprice(id): url="http://quotes.money.163.com/0%s.html"%id txt=requests.get(url).text price=m1.search(txt).group(1) print(id,price) if __name__=="__main__": ts=[] start=time.time() for id in code: t=Thread(target=getprice,args=(id,)) ts.append(t) t.start() for t in ts: t.join() #Wait for the sub thread to run, and the main thread to run again print("Program time consuming:",time.time()-start)
Multi process:
Crawling stock price (multi process version)
#Multi process #Crawling stock prices import requests import re import time from multiprocessing import Process from threading import Thread code=[600016,600000,601939,600036,603683,600050,601890,600795,601857,600584,601231,603165,600644,603005,601198,603690,600643,600131,600776,603609,601377] m1=re.compile(r"price: '(\d\.\d')") def getprice(id): url="http://quotes.money.163.com/0%s.html"%id txt=requests.get(url).text price=m1.search(txt).group(1) print(id,price) ps=[] #Process pool if __name__=="__main__": start=time.time() for id in code: p=Process(target=getprice,args=(id,)) ps.append(p) #Put processes in the list(Process pool) p.start() #Startup process for p in ps: p.join() print(time.time()-start)
Crawling stock price (multi process version) with Pool
#Crawling stock prices import requests import re import time from multiprocessing import Pool #Multi process tape Pool code=[600016,600000,601939,600036,603683,600050,601890,600795,601857,600584,601231,603165,600644,603005,601198,603690,600643,600131,600776,603609,601377] m1=re.compile(r"price: '(\d\.\d')") def getprice(id): url="http://quotes.money.163.com/0%s.html"%id txt=requests.get(url).text price=m1.search(txt).group(1) print(id,price) if __name__=="__main__": start=time.time() p=Pool(4) for id in code: p.apply_async(getprice,args=(id,)) #async Asynchronous, the first parameter is the function name, the second is the parameter of this function p.close() p.join() #Wait for the sub thread to run, and the main thread to run again print("Program time consuming:",time.time()-start)