diff --git a/.gitignore b/.gitignore index a1338d6..6a959a9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ + +# JetBrains +.idea +*.iml + +.build-harness +dist/bin/* diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d9e3865 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,45 @@ +sudo: required + +language: go +go: + - 1.9.x + +addons: + apt: + packages: + - git + - make + - curl + +env: + - DOCKER_IMAGE_NAME=cloudposse/prometheus-to-cloudwatch + +services: +- docker + +install: +- make init +- make travis/docker-login +- make go/deps-build +- make go/deps-dev +- make go-get + +script: +- make go/deps +- make go/test +- make go/lint +- make go/build-all +- ls -l release/ +- make docker/build + +after_success: +- make travis/docker-tag-and-push + +deploy: + - provider: releases + api_key: "$GITHUB_API_KEY" + file_glob: true + file: "release/*" + skip_cleanup: true + on: + tags: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..309fa8f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.10.0 as builder +RUN mkdir -p /go/src/github.com/cloudposse/prometheus-to-cloudwatch +WORKDIR /go/src/github.com/cloudposse/prometheus-to-cloudwatch +COPY . . +RUN go get && CGO_ENABLED=0 go build -v -o "./dist/bin/prometheus-to-cloudwatch" *.go + + +FROM alpine:3.7 +RUN apk add --no-cache ca-certificates +COPY --from=builder /go/src/github.com/cloudposse/prometheus-to-cloudwatch/dist/bin/prometheus-to-cloudwatch /usr/bin/prometheus-to-cloudwatch +ENV PATH $PATH:/usr/bin +ENTRYPOINT ["prometheus-to-cloudwatch"] diff --git a/LICENSE b/LICENSE index 261eeb9..c37833f 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2018 Cloud Posse, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c5b2edf --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +SHELL = /bin/bash + +PATH:=$(PATH):$(GOPATH)/bin + +include $(shell curl --silent -o .build-harness "https://raw.githubusercontent.com/cloudposse/build-harness/master/templates/Makefile.build-harness"; echo .build-harness) + + +.PHONY : go-get +go-get: + go get + + +.PHONY : go-build +go-build: go-get + CGO_ENABLED=0 go build -v -o "./dist/bin/prometheus-to-cloudwatch" *.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..5e1e433 --- /dev/null +++ b/README.md @@ -0,0 +1,212 @@ +# prometheus-to-cloudwatch [![Build Status](https://travis-ci.org/cloudposse/prometheus-to-cloudwatch.svg?branch=master)](https://travis-ci.org/cloudposse/prometheus-to-cloudwatch) + +Utility for scraping Prometheus metrics from a Prometheus client endpoint and publishing them to CloudWatch + + +## Usage + +__NOTE__: The module accepts parameters as command-line arguments or as ENV variables (or any combination of command-line arguments and ENV vars). +Command-line arguments take precedence over ENV vars + + +| Command-line argument | ENV var | Description | +|:-----------------------------|:-----------------------------|:------------------------------------------------------------------------------| +| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics | +| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics | +| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace | +| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region | +| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds | +| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds | +| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from | +| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) | +| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | +| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | + + +__NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`) +or ENV variables (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`), +the chain of credential providers will search for credentials in the shared credential file and EC2 Instance Roles. +This is useful when deploying the module in AWS on Kubernetes with [`kube2iam`](https://github.com/jtblin/kube2iam), +which will provide IAM credentials to containers running inside a Kubernetes cluster, allowing the module to assume an IAM Role with permissions +to publish metrics to CloudWatch. + + +## Examples + +### Build Go program + +```sh +go get + +CGO_ENABLED=0 go build -v -o "./dist/bin/prometheus-to-cloudwatch" *.go +``` + + +### Run locally + +```sh +export AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXXXXX +export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +export CLOUDWATCH_NAMESPACE=kube-state-metrics +export CLOUDWATCH_REGION=us-east-1 +export CLOUDWATCH_PUBLISH_TIMEOUT=5 +export PROMETHEUS_SCRAPE_INTERVAL=30 +export PROMETHEUS_SCRAPE_URL=http://xxxxxxxxxxxx:8080/metrics +export PROMETHEUS_CERT_PATH="" +export PROMETHEUS_KEY_PATH="" +export PROMETHEUS_ACCEPT_INVALID_CERT=true + +./dist/bin/prometheus-to-cloudwatch +``` + + +### Build Docker image +__NOTE__: it will download all `Go` dependencies and then build the program inside the container (see [`Dockerfile`](Dockerfile)) + + +```sh +docker build --tag prometheus-to-cloudwatch --no-cache=true . +``` + + +### Run in a Docker container + +```sh +docker run -i --rm \ + -e AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXXXXX \ + -e AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \ + -e CLOUDWATCH_NAMESPACE=kube-state-metrics \ + -e CLOUDWATCH_REGION=us-east-1 \ + -e CLOUDWATCH_PUBLISH_TIMEOUT=5 \ + -e PROMETHEUS_SCRAPE_INTERVAL=30 \ + -e PROMETHEUS_SCRAPE_URL=http://xxxxxxxxxxxx:8080/metrics \ + -e PROMETHEUS_CERT_PATH="" \ + -e PROMETHEUS_KEY_PATH="" \ + -e PROMETHEUS_ACCEPT_INVALID_CERT=true \ + prometheus-to-cloudwatch +``` + + +### Run on Kubernetes + +To run on `Kubernetes`, we will deploy two [`Helm`](https://helm.sh/) [charts](https://docs.helm.sh/developing_charts/) + +1. [kube-state-metrics](https://github.com/kubernetes/charts/tree/master/stable/kube-state-metrics) - to generates metrics about the state of various objects inside the cluster, such as deployments, nodes and pods + +2. [prometheus-to-cloudwatch](chart) - to scrape metrics from `kube-state-metrics` and publish them to CloudWatch + +Install `kube-state-metrics` chart + +```sh +helm install stable/kube-state-metrics +``` + +Find the running services + +```sh +kubectl get services +``` + +![kube-state-metrics-service](images/kube-state-metrics-service.png) + + +Copy the name of the `kube-state-metrics` service (`gauche-turtle-kube-state-metrics`) into the ENV var `PROMETHEUS_SCRAPE_URL` in [values.yaml](chart/values.yaml). +It should look like this: + +```sh +PROMETHEUS_SCRAPE_URL: "http://gauche-turtle-kube-state-metrics:8080/metrics" +``` + +Deploy `prometheus-to-cloudwatch` chart + +```sh +cd chart +helm install . +``` + +`prometheus-to-cloudwatch` will start scraping the `/metrics` endpoint of the `kube-state-metrics` service and send the Prometheus metrics to CloudWatch + + +![kube-state-metrics-to-cloudwatch](images/kube-state-metrics-to-cloudwatch.png) + + +## Help + +**Got a question?** + +File a GitHub [issue](https://github.com/cloudposse/prometheus-to-cloudwatch/issues), send us an [email](mailto:hello@cloudposse.com) or reach out to us on [Gitter](https://gitter.im/cloudposse/). + + +## Contributing + +### Bug Reports & Feature Requests + +Please use the [issue tracker](https://github.com/cloudposse/prometheus-to-cloudwatch/issues) to report any bugs or file feature requests. + +### Developing + +If you are interested in being a contributor and want to get involved in developing `prometheus-to-cloudwatch`, we would love to hear from you! Shoot us an [email](mailto:hello@cloudposse.com). + +In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. + + 1. **Fork** the repo on GitHub + 2. **Clone** the project to your own machine + 3. **Commit** changes to your own branch + 4. **Push** your work back up to your fork + 5. Submit a **Pull request** so that we can review your changes + +**NOTE:** Be sure to merge the latest from "upstream" before making a pull request! + + +## License + +[APACHE 2.0](LICENSE) © 2018 [Cloud Posse, LLC](https://cloudposse.com) + +See [LICENSE](LICENSE) for full details. + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +## About + +`prometheus-to-cloudwatch` is maintained and funded by [Cloud Posse, LLC][website]. + +![Cloud Posse](https://cloudposse.com/logo-300x69.png) + + +Like it? Please let us know at + +We love [Open Source Software](https://github.com/cloudposse/)! + +See [our other projects][community] +or [hire us][hire] to help build your next cloud platform. + + [website]: https://cloudposse.com/ + [community]: https://github.com/cloudposse/ + [hire]: https://cloudposse.com/contact/ + + +### Contributors + +| [![Erik Osterman][erik_img]][erik_web]
[Erik Osterman][erik_web] | [![Andriy Knysh][andriy_img]][andriy_web]
[Andriy Knysh][andriy_web] | +|-------------------------------------------------------|------------------------------------------------------------------| + + [erik_img]: http://s.gravatar.com/avatar/88c480d4f73b813904e00a5695a454cb?s=144 + [erik_web]: https://github.com/osterman/ + [andriy_img]: https://avatars0.githubusercontent.com/u/7356997?v=4&u=ed9ce1c9151d552d985bdf5546772e14ef7ab617&s=144 + [andriy_web]: https://github.com/aknysh/ diff --git a/chart/.editorconfig b/chart/.editorconfig new file mode 100644 index 0000000..0f17867 --- /dev/null +++ b/chart/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 0000000..b822ab8 --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for prometheus-to-cloudwatch +name: prometheus-to-cloudwatch +version: 0.1.0 diff --git a/chart/README.md b/chart/README.md new file mode 100644 index 0000000..481a050 --- /dev/null +++ b/chart/README.md @@ -0,0 +1,7 @@ +# prometheus-to-cloudwatch + +## Installation + +``` +helm install --name=prometheus-to-cloudwatch --namespace=prometheus-to-cloudwatch +``` diff --git a/chart/charts/common-0.0.1.tgz b/chart/charts/common-0.0.1.tgz new file mode 100644 index 0000000..06ede4f Binary files /dev/null and b/chart/charts/common-0.0.1.tgz differ diff --git a/chart/requirements.lock b/chart/requirements.lock new file mode 100644 index 0000000..c21ab82 --- /dev/null +++ b/chart/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://kubernetes-charts-incubator.storage.googleapis.com/ + version: 0.0.1 +digest: sha256:306378be27e855f6823d057c0d90ba3be227d44dc75b2c70c484a03c2515511e +generated: 2018-03-21T15:44:54.30424-04:00 diff --git a/chart/requirements.yaml b/chart/requirements.yaml new file mode 100644 index 0000000..31bd631 --- /dev/null +++ b/chart/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: +- name: common + version: 0.0.1 + repository: https://kubernetes-charts-incubator.storage.googleapis.com/ diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt new file mode 100644 index 0000000..f77ca0f --- /dev/null +++ b/chart/templates/NOTES.txt @@ -0,0 +1,20 @@ +Make sure to define the following settings in `values.yaml`: + +env: + - CLOUDWATCH_NAMESPACE + - CLOUDWATCH_REGION + - CLOUDWATCH_PUBLISH_TIMEOUT + - PROMETHEUS_SCRAPE_INTERVAL + - PROMETHEUS_SCRAPE_URL + - PROMETHEUS_CERT_PATH + - PROMETHEUS_KEY_PATH + - PROMETHEUS_ACCEPT_INVALID_CERT + +secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + + +If you deploying the module in AWS on Kubernetes with `kube2iam` to assume an IAM Role with permissions +to publish metrics to CloudWatch, don't provide the AWS credentials in 'secrets'. +The chain of credential providers will search for credentials in EC2 Instance Roles. diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml new file mode 100644 index 0000000..24080f4 --- /dev/null +++ b/chart/templates/deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + labels: +{{ include "common.labels.standard" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "common.name" . }} + release: {{ .Release.Name | quote }} + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ template "common.name" . }} + release: {{ .Release.Name | quote }} +{{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 8 }} +{{- end }} + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: +{{- range $name, $value := .Values.env }} +{{- if not (empty $value) }} + - name: {{ $name | quote }} + value: {{ $value | quote }} +{{- end }} +{{- end }} +{{- $secret_name := include "common.fullname" . }} +{{- range $name, $value := .Values.secrets }} +{{- if not ( empty $value) }} + - name: {{ $name | quote }} + valueFrom: + secretKeyRef: + name: {{ $secret_name }} + key: {{ $name | quote }} +{{- end }} +{{- end }} + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} diff --git a/chart/templates/secret.yaml b/chart/templates/secret.yaml new file mode 100644 index 0000000..7e66f4d --- /dev/null +++ b/chart/templates/secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.fullname" . }} + labels: +{{ include "common.labels.standard" . | indent 4 }} +type: Opaque +data: +{{- range $name, $value := .Values.secrets }} +{{- if not (empty $value) }} + {{ $name }}: {{ $value | b64enc }} +{{- end }} +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 0000000..3026645 --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,35 @@ +# Default values for prometheus-to-cloudwatch +# +replicaCount: 1 + +env: + CLOUDWATCH_NAMESPACE: "kube-state-metrics" + CLOUDWATCH_REGION: "us-east-1" + CLOUDWATCH_PUBLISH_TIMEOUT: "5" + PROMETHEUS_SCRAPE_INTERVAL: "20" + PROMETHEUS_SCRAPE_URL: "http://xxxxxxxxxxxx:8080/metrics" + PROMETHEUS_CERT_PATH: "" + PROMETHEUS_KEY_PATH: "" + PROMETHEUS_ACCEPT_INVALID_CERT: "true" + +secrets: + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + +image: + repository: "cloudposse/prometheus-to-cloudwatch" + tag: "0.1.0" + pullPolicy: "IfNotPresent" + +annotations: {} + +resources: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +volume: + emptyDir: {} diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..9ae13dd --- /dev/null +++ b/glide.yaml @@ -0,0 +1,4 @@ +package: github.com/cloudposse/prometheus-to-cloudwatch +import: + - package: github.com/aws/aws-sdk-go/aws + - package: github.com/prometheus diff --git a/images/kube-state-metrics-service.png b/images/kube-state-metrics-service.png new file mode 100644 index 0000000..540600b Binary files /dev/null and b/images/kube-state-metrics-service.png differ diff --git a/images/kube-state-metrics-to-cloudwatch.png b/images/kube-state-metrics-to-cloudwatch.png new file mode 100644 index 0000000..6b8b679 Binary files /dev/null and b/images/kube-state-metrics-to-cloudwatch.png differ diff --git a/main.go b/main.go new file mode 100644 index 0000000..795b1c0 --- /dev/null +++ b/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "flag" + "fmt" + "golang.org/x/net/context" + "log" + "os" + "strconv" + "time" +) + +var ( + awsAccessKeyId = flag.String("aws_access_key_id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS access key Id with permissions to publish CloudWatch metrics") + awsSecretAccessKey = flag.String("aws_secret_access_key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS secret access key with permissions to publish CloudWatch metrics") + cloudWatchNamespace = flag.String("cloudwatch_namespace", os.Getenv("CLOUDWATCH_NAMESPACE"), "CloudWatch Namespace") + cloudWatchRegion = flag.String("cloudwatch_region", os.Getenv("CLOUDWATCH_REGION"), "CloudWatch Region") + cloudWatchPublishTimeout = flag.String("cloudwatch_publish_timeout", os.Getenv("CLOUDWATCH_PUBLISH_TIMEOUT"), "CloudWatch publish timeout in seconds") + prometheusScrapeInterval = flag.String("prometheus_scrape_interval", os.Getenv("PROMETHEUS_SCRAPE_INTERVAL"), "Prometheus scrape interval in seconds") + prometheusScrapeUrl = flag.String("prometheus_scrape_url", os.Getenv("PROMETHEUS_SCRAPE_URL"), "Prometheus scrape URL") + certPath = flag.String("cert_path", os.Getenv("CERT_PATH"), "Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`)") + keyPath = flag.String("key_path", os.Getenv("KEY_PATH"), "Path to Key file (when using SSL for `prometheus_scrape_url`)") + skipServerCertCheck = flag.String("accept_invalid_cert", os.Getenv("ACCEPT_INVALID_CERT"), "Accept any certificate during TLS handshake. Insecure, use only for testing") +) + +func main() { + flag.Parse() + + if *cloudWatchNamespace == "" { + flag.PrintDefaults() + log.Fatal("prometheus-to-cloudwatch: Error: -cloudwatch_namespace or CLOUDWATCH_NAMESPACE required") + } + if *cloudWatchRegion == "" { + flag.PrintDefaults() + log.Fatal("prometheus-to-cloudwatch: Error: -cloudwatch_region or CLOUDWATCH_REGION required") + } + if *prometheusScrapeUrl == "" { + flag.PrintDefaults() + log.Fatal("prometheus-to-cloudwatch: Error: -prometheus_scrape_url or PROMETHEUS_SCRAPE_URL required") + } + if (*certPath != "" && *keyPath == "") || (*certPath == "" && *keyPath != "") { + flag.PrintDefaults() + log.Fatal("prometheus-to-cloudwatch: Error: when using SSL, both -prometheus_cert_path and -prometheus_key_path are required. If not using SSL, do not provide any of them") + } + + var skipCertCheck = true + var err error + + if *skipServerCertCheck != "" { + if skipCertCheck, err = strconv.ParseBool(*skipServerCertCheck); err != nil { + log.Fatal("prometheus-to-cloudwatch: Error: ", err) + } + } + + config := &Config{ + CloudWatchNamespace: *cloudWatchNamespace, + CloudWatchRegion: *cloudWatchRegion, + PrometheusScrapeUrl: *prometheusScrapeUrl, + PrometheusCertPath: *certPath, + PrometheusKeyPath: *keyPath, + PrometheusSkipServerCertCheck: skipCertCheck, + AwsAccessKeyId: *awsAccessKeyId, + AwsSecretAccessKey: *awsSecretAccessKey, + } + + if *prometheusScrapeInterval != "" { + interval, err := strconv.Atoi(*prometheusScrapeInterval) + if err != nil { + log.Fatal("prometheus-to-cloudwatch: error parsing 'prometheus_scrape_interval': ", err) + } + config.CloudWatchPublishInterval = time.Duration(interval) * time.Second + } + + if *cloudWatchPublishTimeout != "" { + timeout, err := strconv.Atoi(*cloudWatchPublishTimeout) + if err != nil { + log.Fatal("prometheus-to-cloudwatch: error parsing 'cloudwatch_publish_timeout': ", err) + } + config.CloudWatchPublishTimeout = time.Duration(timeout) * time.Second + } + + bridge, err := NewBridge(config) + + if err != nil { + log.Fatal("prometheus-to-cloudwatch: Error: ", err) + } + + fmt.Println("prometheus-to-cloudwatch: Starting prometheus-to-cloudwatch bridge") + bridge.Run(context.Background()) +} diff --git a/prometheus_to_cloudwatch.go b/prometheus_to_cloudwatch.go new file mode 100644 index 0000000..9b3374f --- /dev/null +++ b/prometheus_to_cloudwatch.go @@ -0,0 +1,350 @@ +package main + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/matttproud/golang_protobuf_extensions/pbutil" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + "io" + "log" + "mime" + "net/http" + "sort" + "time" +) + +const ( + batchSize = 10 + cwHighResLabel = "__cw_high_res" + cwUnitLabel = "__cw_unit" + acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3` +) + +// Config defines configuration options +type Config struct { + // AWS access key Id with permissions to publish CloudWatch metrics + AwsAccessKeyId string + + // AWS secret access key with permissions to publish CloudWatch metrics + AwsSecretAccessKey string + + // Required. The CloudWatch namespace under which metrics should be published + CloudWatchNamespace string + + // Required. The AWS Region to use + CloudWatchRegion string + + // The frequency with which metrics should be published to Cloudwatch. Default: 15s + CloudWatchPublishInterval time.Duration + + // Timeout for sending metrics to Cloudwatch. Default: 3s + CloudWatchPublishTimeout time.Duration + + // Prometheus scrape URL + PrometheusScrapeUrl string + + // Path to Certificate file + PrometheusCertPath string + + // Path to Key file + PrometheusKeyPath string + + // Accept any certificate during TLS handshake. Insecure, use only for testing + PrometheusSkipServerCertCheck bool +} + +// Bridge pushes metrics to AWS CloudWatch +type Bridge struct { + cloudWatchPublishInterval time.Duration + cloudWatchNamespace string + cw *cloudwatch.CloudWatch + prometheusScrapeUrl string + prometheusCertPath string + prometheusKeyPath string + prometheusSkipServerCertCheck bool +} + +// NewBridge initializes and returns a pointer to a Bridge using the +// supplied configuration, or an error if there is a problem with the configuration +func NewBridge(c *Config) (*Bridge, error) { + b := &Bridge{} + + if c.CloudWatchNamespace == "" { + return nil, errors.New("CloudWatchNamespace required") + } + b.cloudWatchNamespace = c.CloudWatchNamespace + + if c.PrometheusScrapeUrl == "" { + return nil, errors.New("PrometheusScrapeUrl required") + } + b.prometheusScrapeUrl = c.PrometheusScrapeUrl + + b.prometheusCertPath = c.PrometheusCertPath + b.prometheusKeyPath = c.PrometheusKeyPath + b.prometheusSkipServerCertCheck = c.PrometheusSkipServerCertCheck + + if c.CloudWatchPublishInterval > 0 { + b.cloudWatchPublishInterval = c.CloudWatchPublishInterval + } else { + b.cloudWatchPublishInterval = 30 * time.Second + } + + var client = http.DefaultClient + + if c.CloudWatchPublishTimeout > 0 { + client.Timeout = c.CloudWatchPublishTimeout + } else { + client.Timeout = 5 * time.Second + } + + if c.CloudWatchRegion == "" { + return nil, errors.New("CloudWatchRegion required") + } + + config := aws.NewConfig().WithHTTPClient(client).WithRegion(c.CloudWatchRegion) + + // https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html + // https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config + // If credentials are not provided in the variables, the chain of credential providers will search for credentials + // in environment variables, the shared credential file, and EC2 Instance Roles + if c.AwsAccessKeyId != "" && c.AwsSecretAccessKey != "" { + config.Credentials = credentials.NewStaticCredentials(c.AwsAccessKeyId, c.AwsSecretAccessKey, "") + } + + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + + b.cw = cloudwatch.New(sess) + return b, nil +} + +// Run starts a loop that will push metrics to Cloudwatch at the configured interval. Accepts a context.Context to support cancellation +func (b *Bridge) Run(ctx context.Context) { + ticker := time.NewTicker(b.cloudWatchPublishInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + mfChan := make(chan *dto.MetricFamily, 1024) + + go fetchMetricFamilies(b.prometheusScrapeUrl, mfChan, b.prometheusCertPath, b.prometheusKeyPath, b.prometheusSkipServerCertCheck) + + var metricFamilies []*dto.MetricFamily + for mf := range mfChan { + metricFamilies = append(metricFamilies, mf) + } + + err := b.publishMetricsToCloudWatch(metricFamilies) + if err != nil { + log.Println("prometheus-to-cloudwatch: error publishing to CloudWatch:", err) + } else { + log.Println(fmt.Sprintf("prometheus-to-cloudwatch: published %d metrics to CloudWatch", len(metricFamilies))) + } + + case <-ctx.Done(): + log.Println("prometheus-to-cloudwatch: stopping") + return + } + } +} + +// NOTE: The CloudWatch API has the following limitations: +// - Max 40kb request size +// - Single namespace per request +// - Max 10 dimensions per metric +func (b *Bridge) publishMetricsToCloudWatch(mfs []*dto.MetricFamily) error { + vec, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{Timestamp: model.Now()}, mfs...) + + if err != nil { + return err + } + + data := make([]*cloudwatch.MetricDatum, 0, batchSize) + + for _, s := range vec { + name := getName(s.Metric) + data = appendDatum(data, name, s) + + if len(data) == batchSize { + if err := b.flush(data); err != nil { + log.Println("prometheus-to-cloudwatch: error publishing to CloudWatch:", err) + } + data = make([]*cloudwatch.MetricDatum, 0, batchSize) + } + } + + return b.flush(data) +} + +func (b *Bridge) flush(data []*cloudwatch.MetricDatum) error { + if len(data) > 0 { + in := &cloudwatch.PutMetricDataInput{ + MetricData: data, + Namespace: &b.cloudWatchNamespace, + } + _, err := b.cw.PutMetricData(in) + return err + } + return nil +} + +func appendDatum(data []*cloudwatch.MetricDatum, name string, s *model.Sample) []*cloudwatch.MetricDatum { + metric := s.Metric + + if len(metric) == 0 { + return data + } + + d := &cloudwatch.MetricDatum{} + + d.SetMetricName(name). + SetValue(float64(s.Value)). + SetTimestamp(s.Timestamp.Time()). + SetDimensions(getDimensions(metric)). + SetStorageResolution(getResolution(metric)). + SetUnit(getUnit(metric)) + + return append(data, d) +} + +func getName(m model.Metric) string { + if n, ok := m[model.MetricNameLabel]; ok { + return string(n) + } + return "" +} + +// getDimensions returns up to 10 dimensions for the provided metric - one for each label (except the __name__ label) +// If a metric has more than 10 labels, it attempts to behave deterministically and returning the first 10 labels as dimensions +func getDimensions(m model.Metric) []*cloudwatch.Dimension { + if len(m) == 0 { + return make([]*cloudwatch.Dimension, 0) + } else if _, ok := m[model.MetricNameLabel]; len(m) == 1 && ok { + return make([]*cloudwatch.Dimension, 0) + } + + names := make([]string, 0, len(m)) + for k := range m { + if !(k == model.MetricNameLabel || k == cwHighResLabel || k == cwUnitLabel) { + names = append(names, string(k)) + } + } + + sort.Strings(names) + dims := make([]*cloudwatch.Dimension, 0, len(names)) + + for _, name := range names { + if name != "" { + val := string(m[model.LabelName(name)]) + if val != "" { + dims = append(dims, new(cloudwatch.Dimension).SetName(name).SetValue(val)) + } + } + } + + if len(dims) > 10 { + dims = dims[:10] + } + + return dims +} + +// Returns 1 if the metric contains a __cw_high_res label, otherwise returns 60 +func getResolution(m model.Metric) int64 { + if _, ok := m[cwHighResLabel]; ok { + return 1 + } + return 60 +} + +func getUnit(m model.Metric) string { + if u, ok := m[cwUnitLabel]; ok { + return string(u) + } + return "None" +} + +// fetchMetricFamilies retrieves metrics from the provided URL, decodes them into MetricFamily proto messages, and sends them to the provided channel. +// It returns after all MetricFamilies have been sent +func fetchMetricFamilies( + url string, ch chan<- *dto.MetricFamily, + certificate string, key string, + skipServerCertCheck bool, +) { + defer close(ch) + var transport *http.Transport + if certificate != "" && key != "" { + cert, err := tls.LoadX509KeyPair(certificate, key) + if err != nil { + log.Fatal("prometheus-to-cloudwatch: Error: ", err) + } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + InsecureSkipVerify: skipServerCertCheck, + } + tlsConfig.BuildNameToCertificate() + transport = &http.Transport{TLSClientConfig: tlsConfig} + } else { + transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: skipServerCertCheck}, + } + } + client := &http.Client{Transport: transport} + decodeContent(client, url, ch) +} + +func decodeContent(client *http.Client, url string, ch chan<- *dto.MetricFamily) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Fatalf("prometheus-to-cloudwatch: Error: creating GET request for URL %q failed: %s", url, err) + } + req.Header.Add("Accept", acceptHeader) + resp, err := client.Do(req) + if err != nil { + log.Fatalf("prometheus-to-cloudwatch: Error: executing GET request for URL %q failed: %s", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + log.Fatalf("prometheus-to-cloudwatch: Error: GET request for URL %q returned HTTP status %s", url, resp.Status) + } + parseResponse(resp, ch) +} + +// parseResponse consumes an http.Response and pushes it to the channel. +// It returns when all all MetricFamilies are parsed and put on the channel. +func parseResponse(resp *http.Response, ch chan<- *dto.MetricFamily) { + mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + + if err == nil && mediaType == "application/vnd.google.protobuf" && params["encoding"] == "delimited" && params["proto"] == "io.prometheus.client.MetricFamily" { + for { + mf := &dto.MetricFamily{} + if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil { + if err == io.EOF { + break + } + log.Fatalln("prometheus-to-cloudwatch: Error: reading metric family protocol buffer failed:", err) + } + ch <- mf + } + } else { + var parser expfmt.TextParser + metricFamilies, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + log.Fatalln("reading text format failed:", err) + } + for _, mf := range metricFamilies { + ch <- mf + } + } +}