diff --git a/.gitignore b/.gitignore index 6a4b760b2..2aacbc859 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,7 @@ wheelhouse # Lint .pylint + +# Prometheus +.coredns +.prometheus diff --git a/.kokorun/io_cpu.sh b/.kokorun/io_cpu.sh index 9d4070dab..4c3721139 100755 --- a/.kokorun/io_cpu.sh +++ b/.kokorun/io_cpu.sh @@ -50,7 +50,7 @@ bash -x -e tests/test_ignite/start_ignite.sh bash -x -e tests/test_kafka/kafka_test.sh start kafka bash -x -e tests/test_kinesis/kinesis_test.sh start kinesis bash -x -e tests/test_pubsub/pubsub_test.sh start pubsub - +bash -x -e tests/test_prometheus/prometheus_test.sh start export TENSORFLOW_INSTALL="$(python setup.py --package-version)" PYTHON_VERSION=$(python -c 'import sys; print(str(sys.version_info[0]))') @@ -59,6 +59,11 @@ if [[ $PYTHON_VERSION == "2" ]]; then docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:14.04 bash -x -e .travis/python.release.sh "${TENSORFLOW_INSTALL}" python docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:16.04 bash -x -e .travis/wheel.test.sh python + + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:18.04 bash -x -e .travis/wheel.test.sh python ## R @@ -69,13 +74,31 @@ if [[ $PYTHON_VERSION == "2" ]]; then ## TF 2.0 docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:14.04 bash -x -e .travis/python.release.sh "tensorflow==2.0.0b1" --preview ${KOKORO_BUILD_NUMBER} python + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:16.04 bash -x -e .travis/wheel.test.sh python + + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:18.04 bash -x -e .travis/wheel.test.sh python else ## Python 2 docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:14.04 bash -x -e .travis/python.release.sh "${TENSORFLOW_INSTALL}" python3.5 python3.6 + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:16.04 bash -x -e .travis/wheel.test.sh python3.5 + + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:18.04 bash -x -e .travis/wheel.test.sh python3.6 sudo rm -rf dist wheelhouse @@ -83,7 +106,16 @@ else ## TF 2.0 docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:14.04 bash -x -e .travis/python.release.sh "tensorflow==2.0.0b1" --preview ${KOKORO_BUILD_NUMBER} python3.5 python3.6 + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:16.04 bash -x -e .travis/wheel.test.sh python3.5 + + ## Stop then restart prometheus + bash -x -e tests/test_prometheus/prometheus_test.sh stop + bash -x -e tests/test_prometheus/prometheus_test.sh start + docker run -i --rm -v $PWD:/v -w /v --net=host buildpack-deps:18.04 bash -x -e .travis/wheel.test.sh python3.6 fi diff --git a/.travis.yml b/.travis.yml index bd954caa8..5e8df0f32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: bash -x -e tests/test_kafka/kafka_test.sh start kafka bash -x -e tests/test_kinesis/kinesis_test.sh start kinesis bash -x -e tests/test_pubsub/pubsub_test.sh start pubsub + bash -x -e tests/test_prometheus/prometheus_test.sh start - sudo -H python -m pip install -q -U twine --ignore-installed six - twine --version - export TENSORFLOW_INSTALL="$(python setup.py --package-version)" diff --git a/.travis/bazel.configure.sh b/.travis/bazel.configure.sh index 7732eefc1..4f1fa9a74 100755 --- a/.travis/bazel.configure.sh +++ b/.travis/bazel.configure.sh @@ -13,6 +13,8 @@ python get-pip.py -q python -m pip --version if [[ $(uname) == "Darwin" ]]; then python -m pip install -q -U matplotlib numpy --ignore-installed six +else + apt-get -y -qq install git fi python -m pip install -q --ignore-installed six "${TENSORFLOW_INSTALL}" python -c 'import tensorflow as tf; print(tf.version.VERSION)' diff --git a/.travis/python3.7+.release.sh b/.travis/python3.7+.release.sh index 55416ee25..3ad65ae74 100755 --- a/.travis/python3.7+.release.sh +++ b/.travis/python3.7+.release.sh @@ -21,7 +21,7 @@ set -e -x apt-get -y -qq update && apt-get -y -qq install \ software-properties-common \ - gcc g++ make patch \ + gcc g++ make patch git \ unzip curl patchelf add-apt-repository -y ppa:deadsnakes/ppa diff --git a/.travis/wheel.test.sh b/.travis/wheel.test.sh index bb6a9da9c..4a5178bba 100755 --- a/.travis/wheel.test.sh +++ b/.travis/wheel.test.sh @@ -17,7 +17,7 @@ if [[ "$#" -gt 0 ]]; then fi if [[ $(uname) == "Linux" ]]; then apt-get -y -qq update - apt-get -y -qq install $PYTHON_VERSION ffmpeg + apt-get -y -qq install $PYTHON_VERSION ffmpeg dnsutils curl -sSOL https://bootstrap.pypa.io/get-pip.py $PYTHON_VERSION get-pip.py -q fi diff --git a/WORKSPACE b/WORKSPACE index e6d861695..dcedb119d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -436,3 +436,49 @@ http_archive( "https://download.savannah.gnu.org/releases/freetype/freetype-2.10.0.tar.gz", ], ) + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Note: patch is needed as we need to resolve multiple zlib dependencies. +# Patch was created with: +# diff -Naur a b > rules_go.patch +http_archive( + name = "io_bazel_rules_go", + patch_args = ["-p1"], + patches = [ + "//third_party:rules_go.patch", + ], + sha256 = "f04d2373bcaf8aa09bccb08a98a57e721306c8f6043a2a0ee610fd6853dcde3d", + urls = [ + "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/0.18.6/rules_go-0.18.6.tar.gz", + "https://github.com/bazelbuild/rules_go/releases/download/0.18.6/rules_go-0.18.6.tar.gz", + ], +) + +http_archive( + name = "bazel_gazelle", + sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687", + urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"], +) + +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") + +go_rules_dependencies() + +go_register_toolchains() + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") + +gazelle_dependencies() + +go_repository( + name = "com_github_prometheus_common", + importpath = "github.com/prometheus/common", + tag = "v0.4.1", +) + +go_repository( + name = "com_github_prometheus_client_golang", + importpath = "github.com/prometheus/client_golang", + tag = "v0.9.3", +) diff --git a/tensorflow_io/core/BUILD b/tensorflow_io/core/BUILD index 6e8b0aff0..2a16b1de8 100644 --- a/tensorflow_io/core/BUILD +++ b/tensorflow_io/core/BUILD @@ -63,6 +63,7 @@ cc_binary( "//tensorflow_io/hdf5:hdf5_ops", "//tensorflow_io/lmdb:lmdb_ops", "//tensorflow_io/mnist:mnist_ops", + "//tensorflow_io/prometheus:prometheus_ops", "//tensorflow_io/text:text_ops", "@libarchive", "@local_config_tf//:libtensorflow_framework", diff --git a/tensorflow_io/prometheus/BUILD b/tensorflow_io/prometheus/BUILD new file mode 100644 index 000000000..8d6467f0f --- /dev/null +++ b/tensorflow_io/prometheus/BUILD @@ -0,0 +1,26 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "prometheus_ops", + srcs = [ + #"//tensorflow_io/prometheus/go:prometheus.a", + #"//tensorflow_io/prometheus/go:prometheus.h", + "kernels/prometheus_input.cc", + "ops/prometheus_ops.cc", + ], + copts = [ + "-pthread", + "-std=c++11", + "-DNDEBUG", + ], + includes = [ + ".", + ], + linkstatic = True, + deps = [ + "//tensorflow_io/core:dataset_ops", + "//tensorflow_io/prometheus/go:prometheus.cc", + ], +) diff --git a/tensorflow_io/prometheus/__init__.py b/tensorflow_io/prometheus/__init__.py new file mode 100644 index 000000000..7742c1eba --- /dev/null +++ b/tensorflow_io/prometheus/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""PrometheusInput + +@@PrometheusDataset +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow_io.prometheus.python.ops.prometheus_ops import PrometheusDataset + +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + "PrometheusDataset", +] + +remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow_io/prometheus/go/BUILD b/tensorflow_io/prometheus/go/BUILD new file mode 100644 index 000000000..5a55b6552 --- /dev/null +++ b/tensorflow_io/prometheus/go/BUILD @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") + +go_binary( + name = "prometheus", + srcs = ["prometheus.go"], + cgo = True, + linkmode = "c-archive", + visibility = ["//visibility:public"], + deps = [ + "@com_github_prometheus_client_golang//api:go_default_library", + "@com_github_prometheus_client_golang//api/prometheus/v1:go_default_library", + "@com_github_prometheus_common//model:go_default_library", + ], +) diff --git a/tensorflow_io/prometheus/go/prometheus.go b/tensorflow_io/prometheus/go/prometheus.go new file mode 100644 index 000000000..36d9b8774 --- /dev/null +++ b/tensorflow_io/prometheus/go/prometheus.go @@ -0,0 +1,50 @@ +package main + +import "C" + +import ( + "context" + "fmt" + "time" + + "github.com/prometheus/client_golang/api" + "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +//export Query +func Query(endpoint string, query string, sec int64, offset int64, key []int64, val []float64) int64 { + client, err := api.NewClient(api.Config{ + Address: endpoint, + }) + if err != nil { + return -1 + } + value, err := v1.NewAPI(client).Query(context.Background(), query, time.Unix(sec, 0)) + if err != nil { + return -1 + } + if m, ok := value.(model.Matrix); ok && m.Len() > 0 { + index := int64(0) + for index < int64(len(key)) && offset+index < int64(len(m[0].Values)) { + v := m[0].Values[offset+index] + key[index] = v.Timestamp.Unix() + val[index] = float64(v.Value) + index++ + } + return index + } + return 0 +} + +func main() { + key := make([]int64, 20, 20) + val := make([]float64, 20, 20) + sec := time.Now().Unix() + fmt.Println(sec) + returned := Query("http://localhost:9090", "coredns_dns_request_count_total[5m]", sec, 0, key, val) + fmt.Println(returned) + for i := range key { + fmt.Printf("%d, %q, %v\n", i, model.TimeFromUnix(key[i]).Time(), val[i]) + } +} diff --git a/tensorflow_io/prometheus/kernels/prometheus_input.cc b/tensorflow_io/prometheus/kernels/prometheus_input.cc new file mode 100644 index 000000000..9738ac4b3 --- /dev/null +++ b/tensorflow_io/prometheus/kernels/prometheus_input.cc @@ -0,0 +1,83 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "kernels/dataset_ops.h" +#include "tensorflow/core/lib/io/buffered_inputstream.h" + +#include "go/prometheus.h" + +namespace tensorflow { +namespace data { + +class PrometheusState { +public: + PrometheusState() : time_(0), offset_(0) {} + + int64 time_; + int64 offset_; +}; + +class PrometheusInput: public StreamInput { + public: + Status ReadRecord(IteratorContext* ctx, std::unique_ptr& state, int64 record_to_read, int64* record_read, std::vector* out_tensors) const override { + if (state.get() == nullptr) { + state.reset(new PrometheusState()); + state.get()->time_ = time(NULL); + } + Tensor key_tensor(ctx->allocator({}), DT_INT64, {record_to_read}); + Tensor val_tensor(ctx->allocator({}), DT_DOUBLE, {record_to_read}); + GoSlice key_go = {key_tensor.flat().data(), record_to_read, record_to_read}; + GoSlice val_go = {val_tensor.flat().data(), record_to_read, record_to_read}; + GoString endpoint_go = {endpoint().c_str(), static_cast(endpoint().size())}; + GoString query_go = {schema().c_str(), static_cast(schema().size())}; + + GoInt returned = Query(endpoint_go, query_go, state.get()->time_, state.get()->offset_, key_go, val_go); + if (returned < 0) { + return errors::InvalidArgument("prometheus server error: ", returned); + } + if (returned > 0) { + state.get()->offset_ += returned; + *record_read = returned; + if (*record_read < record_to_read) { + Tensor key_tensor_final = key_tensor.Slice(0, *record_read); + Tensor val_tensor_final = val_tensor.Slice(0, *record_read); + out_tensors->emplace_back(std::move(key_tensor_final)); + out_tensors->emplace_back(std::move(val_tensor_final)); + } else { + out_tensors->emplace_back(std::move(key_tensor)); + out_tensors->emplace_back(std::move(val_tensor)); + } + } + return Status::OK(); + } + Status FromEndpoint(const string& endpoint) override { + return Status::OK(); + } + void EncodeAttributes(VariantTensorData* data) const override { + } + bool DecodeAttributes(const VariantTensorData& data) override { + return true; + } + protected: +}; + +REGISTER_UNARY_VARIANT_DECODE_FUNCTION(PrometheusInput, "tensorflow::data::PrometheusInput"); + +REGISTER_KERNEL_BUILDER(Name("PrometheusInput").Device(DEVICE_CPU), + StreamInputOp); +REGISTER_KERNEL_BUILDER(Name("PrometheusDataset").Device(DEVICE_CPU), + StreamInputDatasetOp); +} // namespace data +} // namespace tensorflow diff --git a/tensorflow_io/prometheus/ops/prometheus_ops.cc b/tensorflow_io/prometheus/ops/prometheus_ops.cc new file mode 100644 index 000000000..e47d5ac4a --- /dev/null +++ b/tensorflow_io/prometheus/ops/prometheus_ops.cc @@ -0,0 +1,46 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { + +REGISTER_OP("PrometheusInput") + .Input("source: string") + .Output("handle: variant") + .Attr("filters: list(string) = []") + .Attr("columns: list(string) = []") + .Attr("schema: string = ''") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->MakeShape({c->UnknownDim()})); + return Status::OK(); + }); + +REGISTER_OP("PrometheusDataset") + .Input("input: T") + .Input("batch: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .Attr("T: {string, variant} = DT_VARIANT") + .SetIsStateful() + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->MakeShape({})); + return Status::OK(); + }); + +} // namespace tensorflow diff --git a/tensorflow_io/prometheus/python/__init__.py b/tensorflow_io/prometheus/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tensorflow_io/prometheus/python/ops/__init__.py b/tensorflow_io/prometheus/python/ops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tensorflow_io/prometheus/python/ops/prometheus_ops.py b/tensorflow_io/prometheus/python/ops/prometheus_ops.py new file mode 100644 index 000000000..24792d1e9 --- /dev/null +++ b/tensorflow_io/prometheus/python/ops/prometheus_ops.py @@ -0,0 +1,46 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""PrometheusDataset.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow_io.core.python.ops import data_ops as data_ops +from tensorflow_io.core.python.ops import core_ops as prometheus_ops + +class PrometheusDataset(data_ops.Dataset): + """A Prometheus Dataset + """ + + def __init__(self, endpoint, schema=None, batch=None): + """Create a Prometheus Reader. + + Args: + endpoint: A `tf.string` tensor containing address of + the prometheus server. + schema: A `tf.string` tensor containing the query + string. + batch: Size of the batch. + """ + batch = 0 if batch is None else batch + dtypes = [tf.int64, tf.float64] + shapes = [ + tf.TensorShape([]), tensorflow.TensorShape([])] if batch == 0 else [ + tf.TensorShape([None]), tf.TensorShape([None])] + super(PrometheusDataset, self).__init__( + prometheus_ops.prometheus_dataset, + prometheus_ops.prometheus_input(endpoint, schema=schema), + batch, dtypes, shapes) diff --git a/tests/test_prometheus/prometheus_test.sh b/tests/test_prometheus/prometheus_test.sh new file mode 100644 index 000000000..b190eb2fe --- /dev/null +++ b/tests/test_prometheus/prometheus_test.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e +set -o pipefail + +if [[ "$(uname)" == "Darwin" ]]; then + exit 0 +fi + +action=$1 + +if [ "$action" == "start" ]; then +cat <.coredns +.:1053 { + whoami + prometheus +} +EOF + +docker run -d --rm --name=tensorflow-io-coredns --net=host -v $PWD/.coredns:/Corefile coredns/coredns + +cat <.prometheus +global: + scrape_interval: 1s + evaluation_interval: 1s +alerting: + alertmanagers: + - static_configs: + - targets: +rule_files: +scrape_configs: +- job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] +- job_name: "coredns" + static_configs: + - targets: ['localhost:9153'] +EOF + +docker run -d --rm --name=tensorflow-io-prometheus --net=host -v $PWD/.prometheus:/etc/prometheus/prometheus.yml prom/prometheus + +else +docker rm -f tensorflow-io-coredns +docker rm -f tensorflow-io-prometheus +fi diff --git a/tests/test_prometheus_eager.py b/tests/test_prometheus_eager.py new file mode 100644 index 000000000..9d5610c47 --- /dev/null +++ b/tests/test_prometheus_eager.py @@ -0,0 +1,56 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# ============================================================================== +"""Tests for PrometheusDataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time +import subprocess +import sys +import pytest + +import tensorflow as tf +if not (hasattr(tf, "version") and tf.version.VERSION.startswith("2.")): + tf.compat.v1.enable_eager_execution() +import tensorflow_io.prometheus as prometheus_io # pylint: disable=wrong-import-position + +if sys.platform == "darwin": + pytest.skip( + "prometheus is not supported on macOS yet", allow_module_level=True) + +def test_prometheus_input(): + """test_prometheus_input + """ + for _ in range(6): + subprocess.call(["dig", "@localhost", "-p", "1053", "www.google.com"]) + time.sleep(1) + time.sleep(2) + prometheus_dataset = prometheus_io.PrometheusDataset( + "http://localhost:9090", + schema="coredns_dns_request_count_total[5s]", + batch=2) + i = 0 + for k, v in prometheus_dataset: + print("K, V: ", k.numpy(), v.numpy()) + if i == 4: + # Last entry guaranteed 6.0 + assert v.numpy() == 6.0 + i += 2 + assert i == 6 + +if __name__ == "__main__": + test.main() diff --git a/third_party/rules_go.patch b/third_party/rules_go.patch new file mode 100644 index 000000000..ad2cbac0d --- /dev/null +++ b/third_party/rules_go.patch @@ -0,0 +1,26 @@ +diff -Naur a/go/private/repositories.bzl b/go/private/repositories.bzl +--- a/go/private/repositories.bzl 2019-06-12 15:33:29.000000000 +0000 ++++ b/go/private/repositories.bzl 2019-06-17 04:22:24.220740365 +0000 +@@ -84,22 +84,6 @@ + commit = "582743bf40c5d3639a70f98f183914a2c0cd0680", # v3.7.0, as of 2019-03-03 + shallow_since = "1551387314 -0800", + ) +- # Workaround for protocolbuffers/protobuf#5472 +- # At master, they provide a macro that creates this dependency. We can't +- # load it from here though. +- if "net_zlib" not in native.existing_rules(): +- native.bind( +- name = "zlib", +- actual = "@net_zlib//:zlib", +- ) +- http_archive( +- name = "net_zlib", +- build_file = "@com_google_protobuf//:third_party/zlib.BUILD", +- sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", +- strip_prefix = "zlib-1.2.11", +- urls = ["https://zlib.net/zlib-1.2.11.tar.gz"], +- ) +- + _maybe( + git_repository, + name = "com_github_mwitkow_go_proto_validators",