diff --git a/README.md b/README.md index 5702867..d739a2a 100644 --- a/README.md +++ b/README.md @@ -317,8 +317,10 @@ A collection of delicious docker recipes. - [ ] python - [x] gogs/gogs :cn: - [x] haproxy +- [x] jenkins/jenkins - [x] jmxtrans/jmxtrans - [x] wurstmeister/kafka +- [x] mailgun/kafka-pixy - [x] devopsfaith/krakend - [x] tootsuite/mastodon - [x] netdata/netdata @@ -327,7 +329,6 @@ A collection of delicious docker recipes. - [x] sonatype/nexus3 - [ ] jwilder/nginx-proxy - [x] jazzdd/phpvirtualbox -- [x] jenkins - [x] sonatype/nexus3 - [x] jupyter/notebook - [x] metabase/metabase diff --git a/kafka-pixy/README.md b/kafka-pixy/README.md new file mode 100644 index 0000000..35f7cae --- /dev/null +++ b/kafka-pixy/README.md @@ -0,0 +1,36 @@ +kafka-pixy +========== + +[Kafka-Pixy][1] is a dual API (gRPC and REST) proxy for Kafka with automatic +consumer group control. + +## docker-compose.yml + +```yaml +version: "3.8" +services: + kafka-pixy: + image: mailgun/kafka-pixy + command: --config /etc/kafka-pixy/kafka-pixy.yaml + ports: + - "19091:19091" + - "19092:19092" + volumes: + - ./data:/etc/kafka-pixy + restart: unless-stopped +``` + +## up and running + +```bash +$ mkdir -p data +$ wget -O data/kafka-pixy.yaml https://github.com/mailgun/kafka-pixy/raw/master/default.yaml + +# change kafka & zookeeper settings +$ vim data/kafka-pixy.yaml + +$ docker-compose up -d +$ curl http://127.0.0.1:19092/topics +``` + +[1]: https://github.com/mailgun/kafka-pixy diff --git a/kafka-pixy/data/kafka-pixy.yaml b/kafka-pixy/data/kafka-pixy.yaml new file mode 100644 index 0000000..93de92c --- /dev/null +++ b/kafka-pixy/data/kafka-pixy.yaml @@ -0,0 +1,219 @@ +# TCP address that gRPC API server should listen on. +grpc_addr: 0.0.0.0:19091 + +# TCP address that RESTful API server should listen on. +tcp_addr: 0.0.0.0:19092 + +# Unix domain socket address that RESTful API server should listen on. +# Listening on a unix domain socket is disabled by default. +# unix_addr: "/var/run/kafka-pixy.sock" + +# A map of cluster names to respective proxy configurations. The first proxy +# in the map is considered to be `default`. It is used in API calls that do not +# specify cluster name explicitly. +proxies: + + # Name of a Kafka+ZooKeeper cluster. The only requirement to the name is that + # it should be unique in this config file. The value of this parameter is + # a configuration of a proxy to access the cluster. + default: + + # Unique ID that identifies a Kafka-Pixy instance in both ZooKeeper and + # Kafka. It is automatically generated by default and it is recommended to + # leave it like that. + # client_id: AUTOGENERATED + + # Kafka parameters section. + kafka: + + # List of seed Kafka peers that Kafka-Pixy should access to resolve the + # Kafka cluster topology. + seed_peers: + - kafka:9092 + + # Version of the Kafka cluster. Supported versions are 0.10.2.1 - 2.0.0 + version: 2.0.0 + + # Enable TLS when connecting to the Kafka cluster + tls: false + + # The filepath to the CA root certificate + # ca_certificate_file: + + # The filepath to the client certificate + # client_certificate_file: + + # The filepath to the client certificate key + # client_key_file: + + # Disable hostname verification + # insecure: false + + # Networking parameters section. These all pass through to sarama's + # `config.Net` field. + net: + + # How long to wait for the initial connection. + dial_timeout: 30s + + # How long to wait for a response. + read_timeout: 30s + + # How long to wait for a transmit. + write_timeout: 30s + + # ZooKeeper parameters section. + zoo_keeper: + + # List of seed ZooKeeper peers that Kafka-Pixy should access to resolve the + # ZooKeeper cluster topology. + seed_peers: + - zookeeper:2181 + + # A root directory in ZooKeeper to store consumers data. + # chroot: "" + + # ZooKeeper session timeout has to be a minimum of 2 times the tickTime + # (as set in the server configuration) and a maximum of 20 times the + # tickTime. The default ZooKeeper tickTime is 2 seconds. + # + # See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions + session_timeout: 15s + + # Producer parameters section. + producer: + + # Size of all buffered channels created by the producer module. + channel_buffer_size: 4096 + + # The maximum permitted size of a message (defaults to 1000000). Should be + # set equal to or smaller than the broker's `message.max.bytes`. + max_message_bytes: 1000000 + + # The type of compression to use on messages. Allowed values are: + # none, gzip, snappy, and lz4. + compression: snappy + + # The best-effort number of bytes needed to trigger a flush. + flush_bytes: 1048576 + + # The best-effort frequency of flushes. + flush_frequency: 500ms + + # How long to wait for the cluster to settle between retries. + retry_backoff: 10s + + # The total number of times to retry sending a message before giving up. + retry_max: 6 + + # The level of acknowledgement reliability needed from the broker. + # Allowed values are: + # * no_response: the broker doesn't send any response, the TCP ACK + # is all you get. + # * wait_for_local: the broker responds as soon as it commits to the + # local disk. + # * wait_for_all: the broker waits for all in-sync replicas to commit + # before responding. + required_acks: wait_for_all + + # Period of time that Kafka-Pixy should keep trying to submit buffered + # messages to Kafka. It is recommended to make it large enough to survive + # a ZooKeeper leader election in your setup. + shutdown_timeout: 30s + + # How to assign incoming messages to a Kafka partition. Defaults to using + # a hash of the specified message key, or random if the key is + # unspecified. Allowed values are: + # * hash: for messages with a key, take the FNV-1a hash of the + # bytes, modulus the number of partitions; otherwise use a + # random partition. + # * random: all messages are published to a random partition. + # * roundrobin: iterate over partitions sequentially + partitioner: hash + + # The timeout to specify on individual produce requests to the broker. The + # broker will wait for replication to complete up to this duration before + # returning an error. + timeout: 10s + + # Consumer parameters section. + consumer: + + # If set, Kafka-Pixy will not configure a consumer, and any attempts to + # call the consumer APIs will return an error. + disabled: false + + # Period of time that Kafka-Pixy should wait for an acknowledgement + # before retrying. + ack_timeout: 5m + + # Size of all buffered channels created by the consumer module. + channel_buffer_size: 64 + + # The number of bytes of messages to attempt to fetch for each + # topic-partition in each fetch request. These bytes will be read into + # memory for each partition, so this helps control the memory used by + # the consumer. The fetch request size must be at least as large as + # the maximum message size the server allows or else it is possible + # for the producer to send messages larger than the consumer can fetch. + fetch_max_bytes: 1048576 + + # The maximum amount of time the server will block before answering + # the fetch request if there isn't data immediately available. + fetch_max_wait: 250ms + + # Consume request will wait at most this long until for a message from a + # topic to become available before expiring. + long_polling_timeout: 3s + + # The maximum number of unacknowledged messages allowed for a particular + # group-topic-partition at a time. When this number is reached subsequent + # consume requests will return long polling timeout errors, until some of + # the pending messages are acknowledged. + max_pending_messages: 300 + + # The maximum number of retries Kafka-Pixy will make to offer an + # unack message. Messages that exceeded the number of retries are + # discarded by Kafka-Pixy and acknowledged in Kafka. Zero retries + # means that messages will be offered just once. + # + # If you want Kafka-Pixy to retry indefinitely, then set this + # parameter to -1. + max_retries: -1 + + # How frequently to commit offsets to Kafka. + offsets_commit_interval: 500ms + + # If a request to a Kafka-Pixy fails for any reason, then it should wait this + # long before retrying. + retry_backoff: 500ms + + # Period of time that Kafka-Pixy should keep a subscription for a + # topic by a group in absence of requests to from the consumer group. + subscription_timeout: 15s + +# Configuration for securely accessing the gRPC and web servers +tls: + + # Path to the server certificate file. + # Required if using gRPC SSL/TLS or HTTPS. + # certificate_path: /usr/local/etc/server.crt + + # Path to the server certificate key file. + # Required if using gRPC SSL/TLS or HTTPS. + # key_path: /usr/local/etc/server.key + +# A list of defined loggers, multiple loggers are allowed and each log line will be sent to every logger defined. +logging: + # Logs to stdout in human readable format + - name: console + severity: info + # # Logs to stdout in a JSON format + # - name: json + # severity: info + # # Logs to kafka topic in JSON format + # - name: udplog + # severity: error + # # Logs to syslog + # - name: syslog + # severity: debug diff --git a/kafka-pixy/docker-compose.yml b/kafka-pixy/docker-compose.yml new file mode 100644 index 0000000..a25722a --- /dev/null +++ b/kafka-pixy/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3.8" +services: + kafka-pixy: + image: mailgun/kafka-pixy + command: --config /etc/kafka-pixy/kafka-pixy.yaml + ports: + - "19091:19091" + - "19092:19092" + volumes: + - ./data:/etc/kafka-pixy + restart: unless-stopped