python 53rd Day - Process, Collaboration. select. Asynchronous I/O...

Keywords: Python socket Programming encoding

Process:

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan
 4 import multiprocessing,threading,time
 5 
 6 def run(name):
 7     t=threading.Thread(target=run2)#Create new threads
 8     t.start()
 9     print('process[%s],In printing...'%name)
10     time.sleep(1)
11 
12 def run2():
13     print(threading.get_ident())#Print thread ID
14     time.sleep(2)
15 
16 
17 if __name__ == '__main__':
18     for i in range(10):
19         p=multiprocessing.Process(target=run,args=('The first[%s]A process'%i,))#Create a new process
20         p.start()

Process number:

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan
 4 
 5 import  multiprocessing,threading,time,os
 6 from  multiprocessing import Process#from multprocessing open Process
 7 def info_l(file):
 8     print('Current module name:',__name__)
 9     print('Parent process ID:',os.getppid())
10     print('process ID:',os.getpid())
11     print('\n\n')
12 
13 def f(name):
14     print('See:',name)
15     info_l('Related list')
16 
17 
18 if __name__ == '__main__':
19     info_l('Home Process Related List')
20     p=Process(target=f,args=('Current process',))
21     p.start()
22     p.join()

Process lock:

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan
 4 
 5 from multiprocessing import Process,Lock # Lock Process Lock Communication Middleware
 6 
 7 def f(lock,i):
 8     lock.acquire()#Process lock
 9     print('The first[%s]A process'%i)
10     lock.release()#Unlock
11 if __name__ =='__main__':
12     lock=Lock()#Process Lock Object
13     for i in range(10):
14         p=Process(target=f,args=(lock,i)).start()

Interprocess communication:

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan
 4 
 5 import multiprocessing,queue,threading
 6 from multiprocessing import Process,Queue # Queue Process Communication Middleware
 7 
 8 
 9 '''  Sharing queues between threads
10 def f():
11     q.put([1,None,'Add data'])#queue
12 if __name__ =='__main__':
13     q=queue.Queue()#Thread queue
14     #q=Queue()#Process queue
15     p=threading.Thread(target=f,)#Create thread
16     p.start()
17     print(q.get())#Output
18     p.join()
19 '''
20 def f(q):#Deposit in q object
21     q.put([1,None,'Add data'])#queue
22 if __name__ =='__main__':
23     q=Queue()#Process queue
24     p=Process(target=f,args=(q,))#Create process
25     p.start()
26     print(q.get())#output,Removed
27     p.join()
 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan
 4 import os
 5 from multiprocessing import Process,Pipe,Manager # Pipe Pipeline process communication middleware
 6 
 7 
 8 def f(d,l):
 9     d[os.getpid()]=os.getpid()#Modify dictionary
10     l.append(os.getpid())#Add List Content
11     print(d)
12     print(l)
13 
14 
15 if __name__ =='__main__':
16     with Manager() as manager:
17         d=manager.dict()#Create a modifiable dictionary between processes
18         l=manager.list(range(5))#Create a modifiable list between processes
19         p_list=[]#join Use
20         for i in range(10):
21             p=Process(target=f,args=(d,l))#Create process incoming data,
22             p.start()
23             p_list.append(p)
24         for r in p_list:#Waiting for the process to complete
25             r.join()
26         print(d)
27         print(l)
 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan
 4 
 5 from multiprocessing import Process,Pipe # Pipe Pipeline process communication middleware
 6 
 7 def f(conn):#Deposit in conn object
 8     conn.send(['Subprocesses send information','....'])
 9     print('Receiving information from the parent process:',conn.recv())
10     conn.close()
11 if __name__ =='__main__':
12     parent_conn,child_conn=Pipe()#Generate a pipe,Returns two values,Manage double ends
13     p=Process(target=f,args=(child_conn,))#Create process
14     p.start()
15     print('Receiving information from subprocesses:',parent_conn.recv())
16     parent_conn.send(['Parent process sends information'])
17     p.join()

Process pool:

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #__author__=2017/6/23
 5 import time,os
 6 from multiprocessing import Process,Lock,Pool # Pool Process Pool Communication Middleware
 7 
 8 def Foo(i):
 9     print('The first[%s]A process,ID:'%i,os.getpid())
10     time.sleep(3)
11     return i+100
12 def Bar(arg):
13     print('Callback>>>>:',arg,os.getpid())
14 if __name__ =='__main__':
15     #pool=Pool(processes=5)#Defining a process pool means allowing a process pool to be placed in five processes at the same time
16     pool=Pool(5)#Defining a process pool means allowing a process pool to be placed in five processes at the same time
17     for i in range(10):
18         #pool.apply(func=Foo,args=(i,))#Creating process serial using process pool
19         #pool.apply_async(func=Foo,args=(i,))#Using process pool to create process parallelism
20         pool.apply_async(func=Foo,args=(i,),callback=Bar)#Callback
21 
22     print('End')
23     #pool.join()
24     pool.close()#Must close the process pool first
25     pool.join()#Carry out afterwards join

 

 

Association:

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    10:10
 6 #__author__='Administrator'
 7 
 8 import time
 9 import queue
10 def consumer(name):#Consumer function
11     print('[%s]Consumer Products.......'%name)
12     while True:
13         new_b=yield #Jump point
14         print('[%s] consumption [%s]'%(name,new_b))
15 def producer():#Producer Function
16     r=con.__next__()
17     r2=con2.__next__()
18     n=0
19     while n<10:
20         n+=1
21         con.send(n)#Send to Consumers
22         con2.send(n)
23         print('\033[32;1m[Producer]\033[0m yield a product[%s]'%n)
24 
25 if __name__=='__main__':
26     con=consumer('Consumer A')
27     con2=consumer('Consumer B')
28     p=producer()
 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    10:31
 6 #__author__='Administrator'
 7 #import greenlet
 8 from greenlet import greenlet
 9 
10 def test1():
11     print('Function 1: 12')
12     ger2.switch()#Conduct coprocess switching
13     print('Function 1: 34')
14     ger2.switch()
15 
16 def test2():
17     print('Function two: 56')
18     ger1.switch()
19     print('Function two: 78')
20     ger1.switch()
21 
22 ger1=greenlet(test1)#Create Association
23 ger2=greenlet(test2)
24 ger1.switch()

 

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    10:47
 6 #__author__='Administrator'
 7 
 8 import gevent
 9 def func1():
10     print('func1 reading....')
11     gevent.sleep(2)
12     print('func1 reading two.. end')
13 
14 def func2():
15     print('func2 reading....')
16     gevent.sleep(0)
17     print('func2 reading two.. end')
18 def func3():
19     print('func3 reading....')
20     gevent.sleep(1)
21     print('func3 reading two.. end')
22 
23 gevent.joinall([
24     gevent.spawn(func1),
25     gevent.spawn(func2),
26     gevent.spawn(func3)
27 ])

 

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    14:03
 6 #__author__='Administrator'
 7 from urllib import request
 8 import gevent,time
 9 from gevent import monkey
10 monkey.patch_all()#For all I/O Operation Marking
11 
12 def f(url):#Crawl web page
13     print('Website: ',url)
14     resp=request.urlopen(url)#Open web page
15     data=resp.read()#Read webpage
16     print('Website:[%s]Web data size:[%s]'%(url,len(data)))
17 
18 urls=['https://www.python.org/',
19       'https://hao.360.cn/',
20       'https://www.yahoo.com/']
21 
22 time_start=time.time()#Synchronized serial start time
23 for url in urls:
24     f(url)
25 print('Synchronization duration:',time.time()-time_start)
26 
27 time_start_asy=time.time()#Asynchronous parallel start time
28 gevent.joinall([
29     gevent.spawn(f,'https://www.python.org/'),
30     gevent.spawn(f,'https://hao.360.cn/'),
31     gevent.spawn(f,'https://www.yahoo.com/')
32 ])
33 print('Asynchronous duration:',time.time()-time_start_asy)

 

 

 

Cooperative socket_server implements concurrency

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    14:42
 6 #__author__='Administrator'
 7 
 8 import sys
 9 import socket
10 import time
11 import gevent
12 
13 from gevent import socket,monkey
14 monkey.patch_all()
15 
16 
17 def server(port):
18     s = socket.socket()#socket object
19     s.bind(('0.0.0.0', port))#Server side,bind IP port
20     s.listen(500)
21     print('Monitoring....')
22     while True:
23         cli, addr = s.accept()
24         gevent.spawn(handle_request, cli)#Create a new collaboration
25 
26 
27 
28 def handle_request(conn):
29     try:
30         while True:
31             data = conn.recv(1024)
32             print("recv:", data)
33             conn.send(data)
34             if not data:
35                 conn.shutdown(socket.SHUT_WR)
36 
37     except Exception as  ex:
38         print(ex)
39     finally:
40         conn.close()
41 if __name__ == '__main__':
42     server(8001)

 

select :

socket_server

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    19:34
 6 #__author__='Administrator'
 7 
 8 import select,socket,sys ,queue
 9 
10 s=socket.socket()#Instantiate a connection object
11 s.setblocking(0)#Set to non-blocking
12 server_addr=('localhost',9500)#Setting Binding IP port
13 s.bind(server_addr)#Connection Object Binding IP port
14 s.listen(100)#Queue Connectable Number
15 inputs=[s,]#The first thing to do is to monitor itself.
16 
17 outputs=[]#Send list
18 
19 meg_queues={} #Queue Set Dictionary for Sending Connected Objects
20 
21 while True:
22     print('Monitoring......')
23     readable,writeable,exeptional=select.select(inputs,outputs,inputs)#generate select object,Returns three list connections,Hair pass,error
24 
25     for i in readable: #i For one socket
26         if i is s:#If i yes s Represents a new connection coming in
27             conn,client_addr=i.accept()#Create a new connection
28             print('Access a new connection...',client_addr)
29             conn.setblocking(0)#Also set to non-blocking
30             inputs.append(conn)#join select,Connection List,Avoid blocking
31             meg_queues[conn]=queue.Queue()#Create a queue to add to the dictionary
32         else:
33             try:
34                 data=i.recv(1024)#Receive data if it is not a new connection
35             except Exception as e:
36                 print(e)
37             if data: #If the data is not empty
38                 print('[%s] Data sent [%s]'%(i.getpeername,data))
39                 meg_queues[i].put(data)#Currently connected message queues join data
40                 if i not in outputs:#If the current connection is not in the send list,Join the Send List
41                     outputs.append(i)
42             else:
43                 print('The client has been disconnected....')#Start cleaning up
44                 if i in outputs:#In the Send List
45                     outputs.remove(i)#Delete in the Send List
46                 inputs.remove(i)#Delete in connection list
47                 del meg_queues[i]#Delete in Queue Dictionary
48 
49     for w in writeable:#Circular Send List
50         try:
51             msg=meg_queues[w].get_nowait()#Remove the data from the queue,judge
52         except queue.Empty:#If the data is empty
53             outputs.remove(w)##Delete from the Send List
54         else:
55             w.send(msg)#Send out
56 
57     for e in exeptional:#Loop error list
58         print('Connect[%s]error!'%e.getpeername)
59         inputs.remove(e)##Delete from the Send List
60         if e in outputs:#In the Send List
61             outputs.remove(e)#Delete in the Send List
62         e.close()
63         del meg_queues[e]#Delete in Queue Dictionary

 

 

 

socket_client

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    19:23
 6 #__author__='Administrator'
 7 
 8 import socket
 9 
10 server_addr=('localhost',9500)#Setting Binding IP port
11 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
12 s.connect(server_addr)
13 while True:
14     msg = bytes(input(">>:"),encoding="utf8")#Define a data message
15     if msg:
16         s.sendall(msg)#send data
17     else:
18         print('Can not be empty')
19         continue
20     data = s.recv(1024)#Data collection(Read data)
21     #print(data)
22 
23     print('Received', repr(data.decode()))
24 s.close()

 

selectors :

 1 #!usr/bin/env python
 2 #-*-coding:utf-8-*-
 3 # Author calmyan 
 4 #python 
 5 #2017/6/24    21:58
 6 #__author__='Administrator'
 7 import selectors
 8 import socket
 9 
10 sel = selectors.DefaultSelector()#Generate a create selectors object
11 
12 def accept(sock, mask):
13     conn, addr = sock.accept()  # Create new connections
14     print('accepted', conn, 'from', addr)
15     conn.setblocking(False)#Set it as non-blocking
16     sel.register(conn, selectors.EVENT_READ, read)#Registered Connection,callback read
17 
18 def read(conn, mask):
19     data = conn.recv(1024)  # receive data
20     if data:#Not empty
21         print('echoing', repr(data), 'to', conn)
22         conn.send(data)  # send data
23     else:#If it is empty
24         print('closing', conn)
25         sel.unregister(conn)#Cancellation of registration
26         conn.close()#Close connection
27 
28 server_addr=('localhost',9501)#Setting Binding IP port
29 sock = socket.socket()#Create a sock object
30 sock.bind(server_addr)#binding IP port
31 sock.listen(100)
32 print('Monitoring...')
33 sock.setblocking(False)#Non blocking
34 sel.register(sock, selectors.EVENT_READ, accept)#The registered connection callback function is accept
35 
36 while True:
37     events = sel.select()#Default blocking mode
38     for key, mask in events:#If there is a connection,Access
39         callback = key.data#New connection handle
40         callback(key.fileobj, mask)

 

 

Event Driven and Asynchronous IO

There are several models when the server processes the program of the model:
(1) Every time a request is received, a new process is created to process the request.
(2) Every time a request is received, a new thread is created to process the request.
(3) For each request received, a list of events is placed so that the main process can process the request in a non-blocking I/O manner.
 
The method in (1) will result in poor performance of the server, but its implementation is relatively simple because of the high cost of creating new processes.
The second way, because it involves thread synchronization, may face deadlock And so on.
In the third way, when writing application code, the logic is more complex than the first two.
Considering all factors, it is generally accepted that the third way is the majority. Network Server Approaches adopted
 
Event-driven programming is a programming paradigm where the execution flow of a program is determined by external events. It is characterized by an event cycle, in which callback mechanism is used to trigger corresponding processing when external events occur. Two other common programming paradigms are synchronization (single thread) and multithreaded programming.

Posted by hotmonkey on Mon, 24 Dec 2018 10:00:07 -0800