Elastic search, gorang client aggregation query

Keywords: Go JSON less github

At present, there is a requirement for es aggregation query in the monitoring project, which needs to be implemented in go language,

The requirement is to query that an IP is in a time range, and the average value of each monitoring indicator in a time unit. It's a bit awkward. Here is the query statement of es. It's obvious that the two field s of cpu and mem should be aggregated.
In addition, the time zone must be added, otherwise 8 hours less, you know.

GET /monitor_hf/v1/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "term": {
            "ip": {
              "value": "192.168.1.100"
            }
          }
        },
        {
          "range": {
            "datatime": {
              "gte": 1545634340000,
              "lte": "now"
            }
          }
        }

      ]
    }
  },
  "size": 0,
  "aggs": {
    "AVG_Metric": {
      "date_histogram": {
        "field": "datatime",
        "interval": "day",
        "time_zone": "Asia/Shanghai" ,
         "format": "yyyy-MM-dd HH:mm:ss"
      },
      "aggs": {
        "avg_mem": {
          "avg": {
            "field": "mem"
          }
        },"avg_cpu":{
          "avg": {
            "field": "cpu"
          }
        }
      }
    }
  }
}

The content of a single piece of data is roughly as follows:
datatime is a millisecond time stamp. The mapping of index must be of date type, otherwise aggregation cannot be done.

{
    "cpu":5,
    "mem":77088,
    "datatime":1545702661000,
    "ip":"192.168.1.100"
}

The result of the query looks like this:
As you can see, three pieces of data are found in buckets, meaning of several fields:
Time and millisecond timestamps after key as string and key: format
doc_count: how many docs have been aggregated
AVG? Mem and AVG? CPU: the alias defined in the query statement.

{
  "took": 4,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 2477,
    "max_score": 0,
    "hits": []
  },
  "aggregations": {
    "AVG_Metric": {
      "buckets": [
        {
          "key_as_string": "2018-12-24 00:00:00",
          "key": 1545580800000,
          "doc_count": 402,
          "avg_mem": {
            "value": 71208.1592039801
          },
          "avg_cpu": {
            "value": 4.338308457711443
          }
        },
        {
          "key_as_string": "2018-12-25 00:00:00",
          "key": 1545667200000,
          "doc_count": 1258,
          "avg_mem": {
            "value": 77958.16852146263
          },
          "avg_cpu": {
            "value": 4.639904610492846
          }
        },
        {
          "key_as_string": "2018-12-26 00:00:00",
          "key": 1545753600000,
          "doc_count": 817,
          "avg_mem": {
            "value": 86570.06609547124
          },
          "avg_cpu": {
            "value": 4.975520195838433
          }
        }
      ]
    }
  }
}

Next, use the es client of go language "GitHub. COM / Oliver / elastic" to achieve the same query requirements. I use v6 version:

package main

import (
    "github.com/olivere/elastic"
    "context"
    "encoding/json"
    "fmt"
)

type Aggregations struct {
    AVG_Metric AVG_Metric `json:"AVG_Metric"`
}

type AVG_Metric struct {
    Buckets []Metric `json:"buckets"`
}

type Metric struct {
    Key       int64 `json:"key"`
    Doc_count int64 `json:"doc_count"`
    Avg_mem   Value `json:"avg_mem"`
    Avg_cpu   Value `json:"avg_cpu"`
}

type Value struct {
    Value float64 `json:"value"`
}

var client, _ = elastic.NewSimpleClient(elastic.SetURL("http://127.0.0.1:9200"))

func main() {

    //Specify ip and time range
    boolSearch := elastic.NewBoolQuery().
        Filter(elastic.NewTermsQuery("ip", "192.168.1.100"),elastic.NewRangeQuery("datatime").
        Gte(1545634340000).Lte("now"))

    //Indicators to aggregate
    mem := elastic.NewAvgAggregation().Field("mem")
    cpu := elastic.NewAvgAggregation().Field("cpu")

    //Unit time and specified fields
    aggs := elastic.NewDateHistogramAggregation().
        Interval("day").
        Field("datatime").
        TimeZone("Asia/Shanghai").
        SubAggregation("avg_mem", mem).
        SubAggregation("avg_cpu", cpu)

    //Query statement
    result, _ := client.Search().
        Index("monitor_hf").
        Type("v1").
        Query(boolSearch).
        Size(0).
        Aggregation("AVG_Metric", aggs).
        Do(context.Background())


    //Result output:

    //The first way
    var m *[]Metric
    term, _ := result.Aggregations.Terms("AVG_Metric")
    for _, bucket := range term.Aggregations {
        b, _ := bucket.MarshalJSON()
        //fmt.Println(string(b))
        json.Unmarshal(b, &m)
        for _, v := range *m {
            fmt.Println(v)
        }
    }

    //The second way
    var a *Aggregations
    b, _ := json.Marshal(result.Aggregations)
    //fmt.Println(string(b))
    json.Unmarshal(b, &a)
    for _, v := range a.AVG_Metric.Buckets {
        fmt.Println(v)
    }

}

There are some problems with the output of aggregation results, which can not be found on the Internet, nor can the official find corresponding examples. However, two feasible ones are finally tried out. The first one is recommended, and two structures can be defined less.

Posted by beanwebb on Sat, 30 Nov 2019 15:30:28 -0800