From 708ff765162be713c71a812f91cde8feafbdc9b4 Mon Sep 17 00:00:00 2001 From: kev Date: Sat, 28 Sep 2019 13:06:26 +0800 Subject: [PATCH] update presto --- presto/Dockerfile | 4 +- presto/Dockerfile.alpine | 4 +- presto/README.md | 30 +++++------ .../catalog/jmx.properties | 0 .../catalog/tpcds.properties | 0 .../catalog/tpch.properties | 0 .../{worker1 => worker}/config.properties | 2 +- presto/conf/{worker1 => worker}/jvm.config | 0 .../conf/{worker1 => worker}/log.properties | 0 .../conf/{worker1 => worker}/node.properties | 2 +- presto/docker-compose.yml | 35 ++++++------- presto/docker-stack.yml | 50 +++++++++++++++++++ 12 files changed, 89 insertions(+), 38 deletions(-) rename presto/conf/{worker1 => worker}/catalog/jmx.properties (100%) rename presto/conf/{worker1 => worker}/catalog/tpcds.properties (100%) rename presto/conf/{worker1 => worker}/catalog/tpch.properties (100%) rename presto/conf/{worker1 => worker}/config.properties (84%) rename presto/conf/{worker1 => worker}/jvm.config (100%) rename presto/conf/{worker1 => worker}/log.properties (100%) rename presto/conf/{worker1 => worker}/node.properties (73%) create mode 100644 presto/docker-stack.yml diff --git a/presto/Dockerfile b/presto/Dockerfile index c5dd953..3675ba4 100644 --- a/presto/Dockerfile +++ b/presto/Dockerfile @@ -2,10 +2,10 @@ # Dockerfile for presto # -FROM openjdk:8-jre-slim +FROM openjdk:8-jre-slim-buster MAINTAINER kev -ENV PRESTO_VERSION=0.221 +ENV PRESTO_VERSION=0.226 ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz ENV PRESTO_CLI_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar ENV PRESTO_HOME=/opt/presto diff --git a/presto/Dockerfile.alpine b/presto/Dockerfile.alpine index 0c44338..c1991ae 100644 --- a/presto/Dockerfile.alpine +++ b/presto/Dockerfile.alpine @@ -2,10 +2,10 @@ # Dockerfile for presto # -FROM openjdk:13-alpine +FROM openjdk:8-jre-alpine MAINTAINER kev -ENV PRESTO_VERSION=0.221 +ENV PRESTO_VERSION=0.226 ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz ENV PRESTO_CLI_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar ENV PRESTO_HOME=/opt/presto diff --git a/presto/README.md b/presto/README.md index 7b3f79b..242c81f 100644 --- a/presto/README.md +++ b/presto/README.md @@ -4,31 +4,31 @@ presto [Presto][1] is a distributed SQL query engine designed to query large data sets distributed over one or more heterogeneous data sources. -## docker-compose.yaml - -```yaml -presto: - image: vimagick/presto - ports: - - "8080:8080" - volumes: - - ./conf/standalone:/opt/presto/etc - - ./data/standalone:/data - restart: unless-stopped -``` - ## up and running ```bash -$ docker-comopse up -d +$ ansible all -a 'mkdir -p /data' -$ docker-compose exec presto presto --server localhost:8080 --catalog tpch +$ docker stack deploy -c docker-stack.yml presto + +$ docker service update --replicas-max-per-node=1 presto_worker + +$ docker service update --replicas 10 presto_worker + +$ docker ps | grep presto_coordinator | awk '{print $1}' +4cc5c6c420d7 + +$ docker exec -it 4cc5c6c420d7 presto --server localhost:8080 --catalog tpch >>> show schemas; >>> show tables from tiny; >>> select * from tiny.customer limit 10; >>> quit +$ curl http://localhost:8080/ui/ + $ curl http://localhost:8080/v1/service/presto/general ``` +> :warning: If volume settings are changed, you need to remove them manually on all nodes. + [1]: https://prestodb.io/ diff --git a/presto/conf/worker1/catalog/jmx.properties b/presto/conf/worker/catalog/jmx.properties similarity index 100% rename from presto/conf/worker1/catalog/jmx.properties rename to presto/conf/worker/catalog/jmx.properties diff --git a/presto/conf/worker1/catalog/tpcds.properties b/presto/conf/worker/catalog/tpcds.properties similarity index 100% rename from presto/conf/worker1/catalog/tpcds.properties rename to presto/conf/worker/catalog/tpcds.properties diff --git a/presto/conf/worker1/catalog/tpch.properties b/presto/conf/worker/catalog/tpch.properties similarity index 100% rename from presto/conf/worker1/catalog/tpch.properties rename to presto/conf/worker/catalog/tpch.properties diff --git a/presto/conf/worker1/config.properties b/presto/conf/worker/config.properties similarity index 84% rename from presto/conf/worker1/config.properties rename to presto/conf/worker/config.properties index 30c3fbf..2be3f48 100644 --- a/presto/conf/worker1/config.properties +++ b/presto/conf/worker/config.properties @@ -4,4 +4,4 @@ query.max-memory=50GB query.max-memory-per-node=1GB query.max-total-memory-per-node=2GB discovery.uri=http://coordinator:8080 -node.internal-address=worker1 +#node.internal-address=worker1 diff --git a/presto/conf/worker1/jvm.config b/presto/conf/worker/jvm.config similarity index 100% rename from presto/conf/worker1/jvm.config rename to presto/conf/worker/jvm.config diff --git a/presto/conf/worker1/log.properties b/presto/conf/worker/log.properties similarity index 100% rename from presto/conf/worker1/log.properties rename to presto/conf/worker/log.properties diff --git a/presto/conf/worker1/node.properties b/presto/conf/worker/node.properties similarity index 73% rename from presto/conf/worker1/node.properties rename to presto/conf/worker/node.properties index 32a0bdd..2a9d1e8 100644 --- a/presto/conf/worker1/node.properties +++ b/presto/conf/worker/node.properties @@ -1,3 +1,3 @@ node.environment=production -node.id=worker1 node.data-dir=/data +#node.id=worker1 diff --git a/presto/docker-compose.yml b/presto/docker-compose.yml index a5f89b9..388b06e 100644 --- a/presto/docker-compose.yml +++ b/presto/docker-compose.yml @@ -1,19 +1,20 @@ -coordinator: - image: vimagick/presto +nfs: + image: erichough/nfs-server ports: - - "8080:8080" + - "111:111/tcp" + - "111:111/udp" + - "2049:2049/tcp" + - "2049:2049/udp" + - "32765:32765/tcp" + - "32765:32765/udp" + - "32767:32767/tcp" + - "32767:32767/udp" volumes: - - ./conf/coordinator:/opt/presto/etc - - ./data/coordinator:/data - restart: always - -worker1: - image: vimagick/presto - ports: - - "8081:8080" - volumes: - - ./conf/worker1:/opt/presto/etc - - ./data/worker1:/data - links: - - coordinator - restart: always + - ./conf:/export/presto:ro + - /lib/modules:/lib/modules:ro + environment: + - NFS_LOG_LEVEL=DEBUG + - NFS_EXPORT_0=/export/presto/coordinator *(ro,no_subtree_check) + - NFS_EXPORT_1=/export/presto/worker1 *(ro,no_subtree_check) + privileged: true + restart: unless-stopped diff --git a/presto/docker-stack.yml b/presto/docker-stack.yml new file mode 100644 index 0000000..33fae55 --- /dev/null +++ b/presto/docker-stack.yml @@ -0,0 +1,50 @@ +version: "3.7" + +services: + + coordinator: + image: vimagick/presto + ports: + - "8080:8080" + volumes: + - /data:/data + - presto_coordinator_conf:/opt/presto/etc + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + - node.hostname == presto-coordinator + restart_policy: + condition: on-failure + + worker: + image: vimagick/presto + volumes: + - /data:/data + - presto_worker_conf:/opt/presto/etc + deploy: + replicas: 0 + placement: + constraints: + - node.role == worker + restart_policy: + condition: on-failure + depends_on: + - coordinator + +volumes: + + presto_coordinator_conf: + driver: local + driver_opts: + type: nfs + o: "addr=10.0.0.1,nolock,soft,ro" + device: ":/export/presto/coordinator" + + presto_worker_conf: + driver: local + driver_opts: + type: nfs + o: "addr=10.0.0.1,nolock,soft,ro" + device: ":/export/presto/worker"