Using python3 to grab pinpoint application information

pinpoint api:

• /applications.pinpoint get basic information of applications
• /getAgentList.pinpoint get the corresponding application agent information
• /getServerMapData.pinpoint get the basic data flow information of the corresponding app

​

db.py

import mysql.connector
class MyDB(object):
    """docstring for MyDB"""
    def __init__(self, host, user, passwd , db):
        self.host = host
        self.user = user
        self.passwd = passwd
        self.db = db

        self.connect = None
        self.cursor = None
    def db_connect(self):
        """Database connection
        """
        self.connect = mysql.connector.connect(host=self.host, user=self.user, passwd=self.passwd, database=self.db)
        return self
    def db_cursor(self):
        if self.connect is None:
            self.connect = self.db_connect()

        if not self.connect.is_connected():
            self.connect = self.db_connect()
        self.cursor = self.connect.cursor()
        return self
    def get_rows(self , sql):
        """ Query database results
        :param sql: SQL Sentence
        :param cursor: Database cursor
        """

        self.cursor.execute(sql)
        return self.cursor.fetchall()
    def db_execute(self, sql):
        self.cursor.execute(sql)
        self.connect.commit()
    def db_close(self):
        """Close database connections and cursors
        :param connect: Database connection instance
        :param cursor: Database cursor
        """
        if self.connect:
            self.connect.close()
        if self.cursor:
            self.cursor.close()

pinpoint.py:

 
# -*- coding: utf-8 -*-

'''
Copyright (c) 2018, mersap
All rights reserved.

//Summary: pinpoint.py
//Created by: mersap
//Creation date: January 17, 2019
'''

import sys
import requests
import time
import datetime
import json

sys.path.append('../Golf')
import db #db.py

PPURL = "https://pinpoint.*******.com"


From_Time = datetime.datetime.now() + datetime.timedelta(seconds=-60)
To_Time = datetime.datetime.now()
From_TimeStamp = int(time.mktime(From_Time.timetuple()))*1000
To_TimeStamp = int(time.mktime(datetime.datetime.now().timetuple()))*1000


class PinPoint(object):
    """docstring for PinPoint"""
    def __init__(self, db):
        self.db = db
        super(PinPoint, self).__init__()

    """Obtain pinpoint Application in"""
    def get_applications(self):
        '''return application dict
        '''
        applicationListUrl = PPURL + "/applications.pinpoint"
        res = requests.get(applicationListUrl)
        if res.status_code != 200:
            print("Request exception,Please check.")
            return
        applicationLists = []
        for app in res.json():
            applicationLists.append(app)
        applicationListDict={}
        applicationListDict["applicationList"] = applicationLists
        return applicationListDict
    def getAgentList(self, appname):
        AgentListUrl = PPURL + "/getAgentList.pinpoint"
        param = {
            'application':appname
        }
        res = requests.get(AgentListUrl, params=param)
        if res.status_code != 200:
            print("Request exception,Please check.")
            return
        return len(res.json().keys()),json.dumps(list(res.json().keys()))
        
    def update_servermap(self, appname , from_time=From_TimeStamp,
                         to_time=To_TimeStamp, serviceType='SPRING_BOOT'):
        '''To update app Upstream downstream relationship
        :param appname: apply name
        :param serviceType: Application type
        :param from_time: Starting time
        :param to_time: Termination time
        :
        '''
        #https://pinpoint.*****.com/getServerMapData.pinpoint?applicationName=test-app&from=1547721493000&to=1547721553000&callerRange=1&calleeRange=1&serviceTypeName=TOMCAT&_=1547720614229
        param = {
            'applicationName':appname,
            'from':from_time,
            'to':to_time,
            'callerRange':1,
            'calleeRange':1,
            'serviceTypeName':serviceType
        }

        # serverMapUrl = PPURL + "/getServerMapData.pinpoint"
        serverMapUrl = "{}{}".format(PPURL, "/getServerMapData.pinpoint")
        res = requests.get(serverMapUrl, params=param)
        if res.status_code != 200:
            print("Request exception,Please check.")
            return
        update_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
        links = res.json()["applicationMapData"]["linkDataArray"]
        for link in links :
            ###Exclude the application of test
            if link['sourceInfo']['applicationName'].startswith('test'):
                continue
            #Application name, application type, downstream application name, downstream application type, number of application nodes, number of downstream application nodes, total requests, number of error requests, number of slow requests (the number of this application to the next application)
            application = link['sourceInfo']['applicationName']
            serviceType = link['sourceInfo']['serviceType']
            to_application = link['targetInfo']['applicationName']
            to_serviceType = link['targetInfo']['serviceType']
            agents = len(link.get('fromAgent',' '))
            to_agents =  len(link.get('toAgent',' '))
            totalCount = link['totalCount']
            errorCount = link['errorCount']
            slowCount  = link['slowCount']

            sql = """
                REPLACE into application_server_map (application, serviceType, 
                agents, to_application, to_serviceType, to_agents, totalCount, 
                errorCount,slowCount, update_time, from_time, to_time) 
                VALUES ("{}", "{}", {}, "{}", "{}", {}, {}, {}, {},"{}","{}",
                "{}")""".format(
                    application, serviceType, agents, to_application, 
                    to_serviceType, to_agents, totalCount, errorCount,
                     slowCount, update_time, From_Time, To_Time)
            self.db.db_execute(sql)

    def update_app(self):
        """To update application
        """
        appdict = self.get_applications()
        apps = appdict.get("applicationList")
        update_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
        for app in apps:
            if app['applicationName'].startswith('test'):
                continue
            agents, agentlists = self.getAgentList(app['applicationName'])
            sql = """
                REPLACE  into application_list( application_name, 
                service_type, code, agents, agentlists, update_time) 
                VALUES ("{}", "{}", {}, {}, '{}', "{}");""".format(
                    app['applicationName'], app['serviceType'], 
                    app['code'], agents, agentlists, update_time)
            self.db.db_execute(sql)
        return True

    def update_all_servermaps(self):
        """Update all apps
        """
        appdict = self.get_applications()
        apps = appdict.get("applicationList")
        for app in apps:
            self.update_servermap(app['applicationName'], serviceType=app['serviceType'])
        ###Delete data 7 days ago
        Del_Time = datetime.datetime.now() + datetime.timedelta(days=-7)

        sql = """delete from application_server_map where update_time <= "{}"
        """.format(Del_Time)
        self.db.db_execute(sql)
        return True


def connect_db():
    """ establish SQL Connect
    """
    mydb = db.MyDB(
            host="rm-*****.mysql.rds.aliyuncs.com",
            user="user",
            passwd="passwd",
            db="pinpoint"
            )
    mydb.db_connect()
    mydb.db_cursor()
    return mydb

def main():
    db = connect_db()
    pp = PinPoint(db)
    pp.update_app()
    pp.update_all_servermaps()
    db.db_close()


if __name__ == '__main__':
    main()

• Attach sql statement

CREATE TABLE `application_list` (
  `application_name` varchar(32) NOT NULL,
  `service_type` varchar(32) DEFAULT NULL COMMENT 'Service type',
  `code` int(11) DEFAULT NULL COMMENT 'Service type code',
  `agents` int(11) DEFAULT NULL COMMENT 'agent Number',
  `agentlists` varchar(256) DEFAULT NULL COMMENT 'agent list',
  `update_time` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time',
  PRIMARY KEY (`application_name`),
  UNIQUE KEY `Unique_App` (`application_name`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='pinpoint app list'

CREATE TABLE `application_server_map` (
  `application` varchar(32) NOT NULL COMMENT 'apply name',
  `serviceType` varchar(8) NOT NULL,
  `agents` int(2) NOT NULL COMMENT 'agent Number',
  `to_application` varchar(32) NOT NULL COMMENT 'Downstream service name',
  `to_serviceType` varchar(32) DEFAULT NULL COMMENT 'Downstream service type',
  `to_agents` int(2) DEFAULT NULL COMMENT 'Downstream services agent Number',
  `totalCount` int(8) DEFAULT NULL COMMENT 'Total number of requests',
  `errorCount` int(8) DEFAULT NULL,
  `slowCount` int(8) DEFAULT NULL,
  `update_time` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP,
  `from_time` datetime DEFAULT NULL,
  `to_time` datetime DEFAULT NULL,
  PRIMARY KEY (`application`,`to_application`),
  UNIQUE KEY `Unique_AppMap` (`application`,`to_application`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Application link data'