2 # Copyright (c) 2019 AT&T Intellectual Property.
3 # Copyright (c) 2019 Nokia.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
19 # This script does have the event handler in supervisor to follow the process state.
20 # Main parent process follows the events from the supervised daemon and the child process
21 # provides the process status info via HTTP server interface.
22 # When any of the process state change to PROCESS_STATE_FATAL, then the http server will
23 # respond with status code 500 to indicate faulty operation, normal status code is 200 (working).
24 # Set the script configuration to supervisord.conf as follow:
26 # [eventlistener:process-state]
27 # command=python process-state.py
30 # events=PROCESS_STATE
32 # Following is the example supervisor daemon status for each process.
34 # ver:3.0 server:supervisor serial:16 pool:process-state poolserial:16 eventname:PROCESS_STATE_FATAL len:62
35 # processname:sleep-exit groupname:sleep-exit from_state:BACKOFF
37 # Process states are: PROCESS_STATE_STARTING, PROCESS_STATE_RUNNING, PROCESS_STATE_STOPPING,
38 # PROCESS_STATE_STOPPEDPROCESS_STATE_BACKOFF, PROCESS_STATE_FATAL
50 # needed for python socket library broken pipe WA
51 from multiprocessing import Process, Manager, Value, Lock
57 TABLE_STYLE = ('<style>'
58 'div { font-family: Arial, Helvetica, sans-serif; font-size:8px;}'
59 'p { font-family: Arial, Helvetica, sans-serif;}'
60 'h1 { font-family: Arial, Helvetica, sans-serif; font-size:30px; font-weight: bold; color:#A63434;}'
61 'table, th, td { border: 1px solid black; border-collapse: collapse; font-size:12px;}'
62 'th, td { padding: 3px 10px 3px 10px; text-align: left;}'
63 'th.thr, td.tdr { padding: 3px 10px 3px 10px; text-align: right;}'
64 'th.thc, td.tdc { padding: 3px 10px 3px 10px; text-align: center;}'
65 'table#t1 tr:nth-child(even) { background-color: #eee;}'
66 'table#t1 tr:nth-child(odd) { background-color: #ADC0DC;}'
67 'table#t1 th { background-color: #214D8B; color: white;}</style>')
69 def get_pid_info(pid):
73 process = psutil.Process(pid)
74 # these are the item lists
75 files = process.open_files()
76 # get the open files and connections and count number of fd str
77 sockets = process.connections()
78 descriptors = str(files)+str(sockets)
79 count = descriptors.count("fd=")
80 pdata = {"pid": process.pid,
81 "status": process.status(),
82 "cpu": process.cpu_percent(interval=0.2),
84 "memory": process.memory_info().rss}
85 except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
89 def get_process_html_info():
93 html_data = ("<table width='800px' id='t1'>"
94 "<thead><tr><th>Process</th><th>Date and time</th><th>From state</th><th>to state</th>"
95 "<th class=thc>Pid</th><th class=thc>Fds</th><th class=thc>Mem</th><th class=thc>Cpu</th></tr></thead><tbody>")
96 for proc,data in PROCESS_STATE.items():
101 if data['pid'] is not None:
102 pdata = get_pid_info(data['pid'])
103 if pdata is not None:
105 descriptors = str(pdata['descriptors'])
106 memory_usage = str(pdata['memory']/1024)+" Kb"
107 cpu_usage = str(pdata['cpu'])+" %"
109 '<td>'+str(proc)+'</td>'
110 '<td>'+str(data['time'])+'</td>'
111 '<td>'+str(data['from_state'])+'</td>'
112 '<td>'+str(data['state'])+'</td>'
113 '<td class=tdr>'+str(pid)+'</td>'
114 '<td class=tdr>'+descriptors+'</td>'
115 '<td class=tdr>'+memory_usage+'</td>'
116 '<td class=tdr>'+cpu_usage+'</td>'
119 html_data += ("</tbody></table>")
123 # responds to http request according to the process status
124 class myHTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler):
129 # write HEAD and GET to client and then close
131 s.send_response(HTTP_STATUS['code'])
132 s.send_header("Server-name", "supervisor-process-stalker 1.0")
133 s.send_header("Content-type", "text/html")
138 """Respond to a GET request."""
139 s.send_response(HTTP_STATUS['code'])
140 s.send_header("Server-name", "supervisor-process-stalker 1.0")
141 s.send_header("Content-type", "text/html")
143 html_data = ("<html><head><title>supervisor process event handler</title>"+TABLE_STYLE+
144 "<meta http-equiv='refresh' content='"+str(REFRESH_TIME)+"'/></head>"
145 "<body><h1>Supervisor Process Event Handler</h1>"
146 "<div><table width='800px' id='t1'><tr><td>Status code: "+str(HTTP_STATUS['code'])+"</td></tr></table></div>"
148 s.wfile.write(html_data)
149 html = get_process_html_info()
151 s.wfile.write("</body></html>")
156 # make processing silent - otherwise will mess up the event handler
157 def log_message(self, format, *args):
160 def HTTPServerProcess(address, port, http_status, process_state):
164 # copy the process status global variable
165 PROCESS_STATE = process_state
166 HTTP_STATUS = http_status
168 server = BaseHTTPServer.HTTPServer
170 # redirect stdout to stderr so that the HTTP server won't kill the supervised STDIN/STDOUT interface
171 sys.stdout = sys.stderr
172 # Create a web server and define the handler to manage the
174 server = server((address, port), myHTTPHandler)
175 # Wait forever for incoming http requests
176 server.serve_forever()
178 except KeyboardInterrupt:
179 write_stderr('^C received, shutting down the web server')
180 server.socket.close()
183 for proc,data in d.items():
184 write_stderr(str(proc))
185 for key,val in data.items():
186 write_stderr(str(key)+' is '+str(val))
188 # this is the default logging, only for supervised communication
190 # only eventlistener protocol messages may be sent to stdout
196 # this can be used for debug logging - stdout not allowed
205 # stores the process status info
206 PROCESS_STATE = manager.dict()
207 #HTTP_STATUS_CODE = Value('d', True)
208 HTTP_STATUS = manager.dict()
209 HTTP_STATUS['code'] = 200
211 write_stderr("HTTP STATUS SET TO "+str(HTTP_STATUS['code']))
213 # default http meta key refresh time in seconds
216 # init the default values
217 ADDRESS = "0.0.0.0" # bind to all interfaces
218 PORT = 3000 # web server listen port
219 DEBUG = False # no logging
221 parser = argparse.ArgumentParser()
222 parser.add_argument('--port', dest='port', help='HTTP server listen port, default 3000', required=False, type=int)
223 parser.add_argument('--debug', dest='debug', help='sets the debug mode for logging', required=False, action='store_true')
224 parser.add_argument('--address', dest='address', help='IP listen address (e.g. 172.16.0.3), default all interfaces', required=False, type=str)
225 parser.add_argument('--refresh', dest='refresh', help='HTTP auto refresh time in second default is 3 seconds', required=False, type=int)
226 args = parser.parse_args()
228 if args.port is not None:
230 if args.address is not None:
231 ADDRESS = args.address
232 if args.debug is not False:
235 # Start the http server, bind to address
236 httpServer = Process(target=HTTPServerProcess, args=(ADDRESS, PORT, HTTP_STATUS, PROCESS_STATE))
239 # set the signal handler this phase
240 signal.signal(signal.SIGQUIT, doExit)
241 signal.signal(signal.SIGTERM, doExit)
242 signal.signal(signal.SIGINT, doExit)
243 signal.signal(signal.SIGCLD, doExit)
245 while httpServer.is_alive():
246 # transition from ACKNOWLEDGED to READY
248 write_stdout('READY\n')
249 # read header line and print it to stderr
250 line = sys.stdin.readline()
253 # read event payload and print it to stderr
254 headers = dict([ x.split(':') for x in line.split() ])
255 process_state = headers['eventname']
257 if process_state == 'PROCESS_STATE_FATAL':
258 write_stderr('Status changed to FATAL')
259 HTTP_STATUS['code'] = 500
261 short_state = process_state.replace('PROCESS_STATE_', '')
262 length = int(headers['len'])
263 data = sys.stdin.read(length)
266 process = dict([ x.split(':') for x in data.split() ])
269 pid = int(process['pid'])
270 now = datetime.datetime.now()
271 timestamp=str(now.strftime("%Y/%m/%d %H:%M:%S"))
272 PROCESS_STATE[process['processname']] = {'time': timestamp, 'state': short_state, 'from_state': process['from_state'],
274 # transition from READY to ACKNOWLEDGED
275 write_stdout('RESULT 2\nOK')
278 def kill_child_processes():
279 procs = psutil.Process().children()
284 except psutil.NoSuchProcess:
287 def doExit(signalNumber, frame):
288 write_stderr("Got signal: "+str(signalNumber)+" need to exit ...")
289 kill_child_processes()
292 if __name__ == '__main__':