2 # ============LICENSE_START===============================================
3 # Copyright (C) 2020 Nordix Foundation. All rights reserved.
4 # ========================================================================
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 # ============LICENSE_END=================================================
19 from flask import Flask, request
20 from time import sleep
24 from flask import Flask
25 from flask import Response
27 from threading import RLock, Thread
32 # Disable all logging of GET on reading counters
33 class AjaxFilter(logging.Filter):
34 def filter(self, record):
35 return ("/counter/" not in record.getMessage())
37 log = logging.getLogger('werkzeug')
38 log.addFilter(AjaxFilter())
42 # list of messages to/from Dmaap
53 cntr_msg_requests_submitted=0
54 cntr_msg_requests_fetched=0
55 cntr_msg_responses_submitted=0
56 cntr_msg_responses_fetched=0
58 # Request and response constants
59 ORU_WRITE_URL="/events/unauthenticated.SEC_FAULT_OUTPUT"
60 ORU_READ_URL="/events/unauthenticated.SEC_FAULT_OUTPUT/users/test/"
61 AGENT_WRITE_URL="/events/A1-POLICY-AGENT-WRITE"
62 AGENT_READ_URL="/events/A1-POLICY-AGENT-READ/users/policy-agent"
63 APP_WRITE_URL="/send-request"
64 APP_READ_URL="/receive-response"
65 MIME_TEXT="text/plain"
66 MIME_JSON="application/json"
67 CAUGHT_EXCEPTION="Caught exception: "
68 SERVER_ERROR="Server error :"
72 generic_topics_upload_baseurl=""
75 downloader_thread=None
76 generic_uploader_thread=None
78 # Function to upload PMS messages to dmaap
81 global cntr_msg_requests_fetched
83 print("Starting uploader")
85 headers = {'Content-type': 'application/json', 'Accept': '*/*'}
86 #url="http://"+topic_host+"/events/"+topic_read
90 while (len(msg_requests)>0):
94 print("Sending to dmaap : "+ url)
95 print("Sending to dmaap : "+ msg)
96 resp=requests.post(url, data=msg, headers=headers, timeout=10)
97 if (resp.status_code<199 & resp.status_code > 299):
98 print("Failed, response code: " + str(resp.status_code))
101 print("Dmaap response code: " + str(resp.status_code))
102 print("Dmaap response text: " + str(resp.text))
105 cntr_msg_requests_fetched += 1
106 except Exception as e:
107 print("Failed, exception: "+ str(e))
112 # Function to download PMS messages from dmaap
113 def dmaap_downloader():
115 global cntr_msg_responses_submitted
117 print("Starting uploader")
122 #url="http://"+topic_host+"/events/"+topic_write+"/users/mr-stub?timeout=15000&limit=100"
124 headers = {'Accept': 'application/json'}
125 print("Reading from dmaap: " + url)
126 resp=requests.get(url, headers=headers)
127 if (resp.status_code<199 & resp.status_code > 299):
128 print("Failed, response code: " + resp.status_code)
131 print("Recieved data from dmaap mr")
134 print("Recieved data (raw): " + str(resp.text))
135 if isinstance(data, list):
137 item=json.loads(item)
138 corrid=str(item["correlationId"])
139 status=str(item["status"])
140 msg=str(item["message"])
141 item_str=msg+status[0:3]
143 msg_responses[corrid]=item_str
144 cntr_msg_responses_submitted += 1
146 print("Data from dmaap is not json array: " + str(resp.text))
148 except Exception as e:
149 print("Corrupt data from dmaap mr - dropping " + str(data))
150 print("CAUGHT_EXCEPTION" + str(e) + " "+traceback.format_exc())
152 except Exception as e:
155 # Function to upload generic messages to dmaap
156 def dmaap_generic_uploader():
158 global cntr_msg_requests_fetched
160 print("Starting generic uploader")
162 headers_json = {'Content-type': 'application/json', 'Accept': '*/*'}
163 headers_text = {'Content-type': 'text/plain', 'Accept': '*/*'}
166 if (len(generic_messages)):
167 keys_copy = list(generic_messages.keys())
168 for topicname in keys_copy: #topicname contains the path of the topics, eg. "/event/<topic>"
169 topic_queue=generic_messages[topicname]
170 if (len(topic_queue)>0):
171 if (topicname.endswith(".text")):
178 url=generic_topics_upload_baseurl+topicname
179 print("Sending to dmaap : "+ url)
180 print("Sending to dmaap : "+ msg)
181 print("Sending to dmaap : "+ str(headers))
183 resp=requests.post(url, data=msg, headers=headers, timeout=10)
184 if (resp.status_code<199 & resp.status_code > 299):
185 print("Failed, response code: " + str(resp.status_code))
188 print("Dmaap response code: " + str(resp.status_code))
189 print("Dmaap response text: " + str(resp.text))
192 cntr_msg_requests_fetched += 1
193 except Exception as e:
194 print("Failed, exception: "+ str(e))
205 # Helper function to create a Dmaap PMS request message
206 # args : <GET|PUT|DELETE> <correlation-id> <json-string-payload - may be None> <url>
207 # response: json formatted string of a complete Dmaap message
208 def create_message(operation, correlation_id, payload, url):
209 if (payload is None):
211 time_stamp=datetime.datetime.utcnow()
212 msg = '{\"apiVersion\":\"1.0\",\"operation\":\"'+operation+'\",\"correlationId\":\"'+correlation_id+'\",\"originatorId\": \"849e6c6b420\",'
213 msg = msg + '\"payload\":'+payload+',\"requestId\":\"23343221\", \"target\":\"policy-agent\", \"timestamp\":\"'+str(time_stamp)+'\", \"type\":\"request\",\"url\":\"'+url+'\"}'
217 ### MR-stub interface, for MR control
219 # Send a PMS message to MR
220 # URI and parameters (PUT or POST): /send-request?operation=<GET|PUT|POST|DELETE>&url=<url>
221 # response: <correlation-id> (http 200) o4 400 for parameter error or 500 for other errors
222 @app.route(APP_WRITE_URL,
223 methods=['PUT','POST'])
226 global cntr_msg_requests_submitted
228 print("APP_WRITE_URL lock")
230 oper=request.args.get('operation')
232 print(APP_WRITE_URL+" parameter 'operation' missing")
233 return Response('Parameter operation missing in request', status=400, mimetype=MIME_TEXT)
235 url=request.args.get('url')
237 print(APP_WRITE_URL+" parameter 'url' missing")
238 return Response('Parameter url missing in request', status=400, mimetype=MIME_TEXT)
240 if (oper != "GET" and oper != "PUT" and oper != "POST" and oper != "DELETE"):
241 print(APP_WRITE_URL+" parameter 'operation' need to be: DEL|PUT|POST|DELETE")
242 return Response('Parameter operation does not contain DEL|PUT|POST|DELETE in request', status=400, mimetype=MIME_TEXT)
244 print(APP_WRITE_URL+" operation="+oper+" url="+url)
245 correlation_id=str(time.time_ns())
247 if (oper == "PUT") and (request.json is not None):
248 payload=json.dumps(request.json)
250 msg=create_message(oper, correlation_id, payload, url)
252 print(APP_WRITE_URL+" MSG(correlationid = "+correlation_id+"): " + json.dumps(json.loads(msg), indent=2))
253 msg_requests.append(msg)
254 cntr_msg_requests_submitted += 1
255 return Response(correlation_id, status=200, mimetype=MIME_TEXT)
256 except Exception as e:
257 print(APP_WRITE_URL+"-"+CAUGHT_EXCEPTION+" "+str(e) + " "+traceback.format_exc())
258 return Response(SERVER_ERROR+" "+str(e), status=500, mimetype=MIME_TEXT)
260 # Receive a PMS message response for MR for the included correlation id
261 # URI and parameter, (GET): /receive-response?correlationid=<correlation-id>
262 # response: <json-array of 1 response> 200 or empty 204 or other errors 500
263 @app.route(APP_READ_URL,
265 def receiveresponse():
267 global cntr_msg_responses_fetched
269 print("APP_READ_URL lock")
271 cid=request.args.get('correlationid')
273 print(APP_READ_URL+" parameter 'correclationid' missing")
274 return Response('Parameter correlationid missing in json', status=500, mimetype=MIME_TEXT)
276 if (cid in msg_responses):
277 answer=msg_responses[cid]
278 del msg_responses[cid]
279 print(APP_READ_URL+" response (correlationid="+cid+"): " + answer)
280 cntr_msg_responses_fetched += 1
281 return Response(answer, status=200, mimetype=MIME_JSON)
283 print(APP_READ_URL+" - no messages (correlationid="+cid+"): ")
284 return Response('', status=204, mimetype=MIME_JSON)
285 except Exception as e:
286 print(APP_READ_URL+"-"+CAUGHT_EXCEPTION+" "+str(e) + " "+traceback.format_exc())
287 return Response(SERVER_ERROR+" "+str(e), status=500, mimetype=MIME_TEXT)
289 ### Dmaap interface ###
291 # Read PMS messages stream. URI according to agent configuration.
292 # URI, (GET): /events/A1-POLICY-AGENT-READ/users/policy-agent
293 # response: 200 <json array of request messages>, or 500 for other errors
294 @app.route(AGENT_READ_URL,
298 global cntr_msg_requests_fetched
300 if topic_write or topic_read:
301 return Response('Url not available when running as mrstub frontend', status=404, mimetype=MIME_TEXT)
303 limit=request.args.get('limit')
312 print("Limting number of returned messages to: "+str(limit))
314 timeout=request.args.get('timeout')
315 if (timeout is None):
318 timeout=min(int(timeout),60000)
320 start_time=int(round(time.time() * 1000))
321 current_time=int(round(time.time() * 1000))
323 while(current_time<start_time+int(timeout)):
325 if(len(msg_requests)>0):
329 while(cntr<limit and len(msg_requests)>0):
332 msgs=msgs+msg_requests.pop(0)
333 cntr_msg_requests_fetched += 1
336 print(AGENT_READ_URL+" MSGs: "+json.dumps(json.loads(msgs), indent=2))
337 return Response(msgs, status=200, mimetype=MIME_JSON)
338 except Exception as e:
339 print(AGENT_READ_URL+"-"+CAUGHT_EXCEPTION+" "+str(e) + " "+traceback.format_exc())
340 return Response(SERVER_ERROR+" "+str(e), status=500, mimetype=MIME_TEXT)
341 sleep(0.025) # sleep 25 milliseconds
342 current_time=int(round(time.time() * 1000))
344 print("timeout: "+str(timeout)+", start_time: "+str(start_time)+", current_time: "+str(current_time))
345 return Response("[]", status=200, mimetype=MIME_JSON)
347 # Write PMS messages stream. URI according to agent configuration.
348 # URI and payload, (PUT or POST): /events/A1-POLICY-AGENT-WRITE <json array of response messages>
349 # response: OK 200 or 400 for missing json parameters, 500 for other errors
350 @app.route(AGENT_WRITE_URL,
351 methods=['PUT','POST'])
354 global cntr_msg_responses_submitted
356 if topic_write or topic_read:
357 return Response('Url not available when running as mrstub frontend', status=404, mimetype=MIME_TEXT)
360 print("AGENT_WRITE_URL lock")
363 print(AGENT_WRITE_URL+ " json=" + json.dumps(answer, indent=2))
364 if isinstance(answer, dict):
365 #Create a an array if the answer is a dict (single message)
367 answer_list.append(answer)
371 cid=item['correlationId']
373 print(AGENT_WRITE_URL+" parameter 'correlatonid' missing")
374 return Response('Parameter <correlationid> missing in json', status=400, mimetype=MIME_TEXT)
377 print(AGENT_WRITE_URL+" parameter 'msgs' missing")
378 return Response('Parameter >message> missing in json', status=400, mimetype=MIME_TEXT)
379 status=item['status']
381 print(AGENT_WRITE_URL+" parameter 'status' missing")
382 return Response('Parameter <status> missing in json', status=400, mimetype=MIME_TEXT)
383 if isinstance(msg, list) or isinstance(msg, dict):
384 msg_str=json.dumps(msg)+status[0:3]
386 msg_str=msg+status[0:3]
387 msg_responses[cid]=msg_str
388 cntr_msg_responses_submitted += 1
389 print(AGENT_WRITE_URL+ " msg+status (correlationid="+cid+") :" + str(msg_str))
390 except Exception as e:
391 print(AGENT_WRITE_URL+"-"+CAUGHT_EXCEPTION+" "+str(e) + " "+traceback.format_exc())
392 return Response('{"message": "' + SERVER_ERROR + ' ' + str(e) + '","status":"500"}', status=200, mimetype=MIME_JSON)
394 return Response('{}', status=200, mimetype=MIME_JSON)
396 @app.route(ORU_WRITE_URL,
397 methods=['PUT','POST'])
400 msg=json.dumps(request.json)
401 msg_requests.append(msg)
402 return Response('{}', status=200, mimetype=MIME_JSON)
404 @app.route(ORU_READ_URL,
408 if(len(msg_requests)>0):
409 rsp=msg_requests.pop(0)
412 return Response(json.dumps(res), status=200, mimetype=MIME_JSON)
413 return Response("[]", status=200, mimetype=MIME_JSON)
415 # Generic POST catching all urls starting with /events/<topic>.
416 # Writes the message in a que for that topic
417 @app.route("/events/<path>",
419 def generic_write(path):
420 global generic_messages
421 global cntr_msg_responses_submitted
422 urlkey="/events/"+str(path)
423 write_method=str(request.method)
426 if (urlkey.endswith(".text")):
427 payload=str(request.data.decode('UTF-8'))
428 print(write_method+" on "+urlkey+" text=" + payload)
431 print(write_method+" on "+urlkey+" json=" + json.dumps(payload))
433 if (urlkey in generic_messages.keys()):
434 topicmsgs=generic_messages[urlkey]
436 generic_messages[urlkey]=topicmsgs
438 if isinstance(payload, list):
439 for listitem in payload:
440 topicmsgs.append(listitem)
442 topicmsgs.append(payload)
444 cntr_msg_responses_submitted += 1
445 except Exception as e:
446 print(write_method + "on "+urlkey+"-"+CAUGHT_EXCEPTION+" "+str(e) + " "+traceback.format_exc())
447 return Response('{"message": "' + SERVER_ERROR + ' ' + str(e) + '","status":"500"}', status=200, mimetype=MIME_JSON)
449 return Response('{}', status=200, mimetype=MIME_JSON)
451 # Generic GET catching all urls starting with /events/. Returns max 4096 json msgs in an array.
452 # Returns only the messages previously written to the same urls
453 @app.route("/events/<path:path>",
455 def generic_read(path):
456 global generic_messages
457 global cntr_msg_requests_fetched
459 if generic_topics_upload_baseurl:
460 return Response('Url not available when running as mrstub frontend', status=404, mimetype=MIME_TEXT)
462 urlpath="/events/"+str(path)
463 urlkey="/events/"+str(path).split("/")[0] #Extract topic
464 print("GET on topic"+urlkey)
465 limit=request.args.get('limit')
474 print("Limting number of returned messages to: "+str(limit))
476 timeout=request.args.get('timeout')
477 if (timeout is None):
480 timeout=min(int(timeout),60000)
482 start_time=int(round(time.time() * 1000))
483 current_time=int(round(time.time() * 1000))
485 if (urlkey in generic_messages.keys()):
486 topicmsgs=generic_messages[urlkey]
488 while(current_time<start_time+int(timeout)):
490 if(len(topicmsgs)>0):
494 while(cntr<limit and len(topicmsgs)>0):
497 msgs=msgs+json.dumps(json.dumps(topicmsgs.pop(0)))
498 cntr_msg_requests_fetched += 1
501 print("GET on "+urlpath+" MSGs: "+msgs)
502 return Response(msgs, status=200, mimetype=MIME_JSON)
503 except Exception as e:
504 print("GET on "+urlpath+"-"+CAUGHT_EXCEPTION+" "+str(e) + " "+traceback.format_exc())
505 return Response(SERVER_ERROR+" "+str(e), status=500, mimetype=MIME_TEXT)
506 sleep(0.025) # sleep 25 milliseconds
507 current_time=int(round(time.time() * 1000))
509 print("timeout: "+str(timeout)+", start_time: "+str(start_time)+", current_time: "+str(current_time))
510 return Response("[]", status=200, mimetype=MIME_JSON)
513 ### Functions for metrics read out ###
515 @app.route('/counter/requests_submitted',
517 def requests_submitted():
518 return Response(str(cntr_msg_requests_submitted), status=200, mimetype=MIME_TEXT)
520 @app.route('/counter/requests_fetched',
522 def requests_fetched():
523 return Response(str(cntr_msg_requests_fetched), status=200, mimetype=MIME_TEXT)
525 @app.route('/counter/responses_submitted',
527 def responses_submitted():
528 return Response(str(cntr_msg_responses_submitted), status=200, mimetype=MIME_TEXT)
530 @app.route('/counter/responses_fetched',
532 def responses_fetched():
533 return Response(str(cntr_msg_responses_fetched), status=200, mimetype=MIME_TEXT)
535 @app.route('/counter/current_requests',
537 def current_requests():
538 return Response(str(len(msg_requests)), status=200, mimetype=MIME_TEXT)
540 @app.route('/counter/current_responses',
542 def current_responses():
543 return Response(str(len(msg_responses)), status=200, mimetype=MIME_TEXT)
547 # Reset all messsages and counters
549 methods=['GET', 'POST', 'PUT'])
551 global cntr_msg_requests_submitted
552 global cntr_msg_requests_fetched
553 global cntr_msg_responses_submitted
554 global cntr_msg_responses_fetched
558 cntr_msg_requests_submitted=0
559 cntr_msg_requests_fetched=0
560 cntr_msg_responses_submitted=0
561 cntr_msg_responses_fetched=0
564 return Response('OK', status=200, mimetype=MIME_TEXT)
566 # Get env vars, if present
567 if os.getenv("TOPIC_READ") is not None:
569 print("Env variables:")
570 print("TOPIC_READ:"+os.environ['TOPIC_READ'])
571 print("TOPIC_WRITE:"+os.environ['TOPIC_WRITE'])
573 topic_read=os.environ['TOPIC_READ']
574 topic_write=os.environ['TOPIC_WRITE']
577 if topic_read and downloader_thread is None:
578 downloader_thread=Thread(target=dmaap_downloader)
579 downloader_thread.start()
581 if topic_write and uploader_thread is None:
582 uploader_thread=Thread(target=dmaap_uploader)
583 uploader_thread.start()
585 if os.environ['GENERIC_TOPICS_UPLOAD_BASEURL'] is not None:
586 print("GENERIC_TOPICS_UPLOAD_BASEURL:"+os.environ['GENERIC_TOPICS_UPLOAD_BASEURL'])
587 generic_topics_upload_baseurl=os.environ['GENERIC_TOPICS_UPLOAD_BASEURL']
588 if generic_topics_upload_baseurl and generic_uploader_thread is None:
589 generic_uploader_thread=Thread(target=dmaap_generic_uploader)
590 generic_uploader_thread.start()
592 if os.getenv("TOPIC_READ") is None or os.environ['GENERIC_TOPICS_UPLOAD_BASEURL'] is None:
593 print("No env variables - OK")
595 if __name__ == "__main__":
596 app.run(port=HOST_PORT, host=HOST_IP)