diff --git a/scrapyd-onbuild/README.md b/scrapyd-onbuild/README.md deleted file mode 100644 index e508a47..0000000 --- a/scrapyd-onbuild/README.md +++ /dev/null @@ -1,5 +0,0 @@ -scrapyd-onbuild -=============== - -Dockerfile for building an image that runs [scrapyd][1]. - diff --git a/scrapyd-onbuild/requirements.txt b/scrapyd-onbuild/requirements.txt deleted file mode 100644 index 658130b..0000000 --- a/scrapyd-onbuild/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -psycopg2 diff --git a/scrapyd-onbuild/Dockerfile b/scrapyd.old/Dockerfile similarity index 86% rename from scrapyd-onbuild/Dockerfile rename to scrapyd.old/Dockerfile index ed92bcd..860b687 100644 --- a/scrapyd-onbuild/Dockerfile +++ b/scrapyd.old/Dockerfile @@ -1,10 +1,13 @@ # -# Dockerfile for scrapyd-onbuild +# Dockerfile for scrapyd # FROM debian:jessie MAINTAINER kev +ADD ./scrapyd.conf /etc/scrapyd/ +ADD ./requirements.txt /etc/scrapyd/ + RUN apt-get update \ && apt-get install -y autoconf \ build-essential \ @@ -36,6 +39,7 @@ RUN apt-get update \ && rm -rf libxslt \ && curl -sSL https://bootstrap.pypa.io/get-pip.py | python \ && pip install scrapy==0.24.6 scrapyd==1.0.1 \ + && pip install -r /etc/scrapyd/requirements.txt \ && curl -sSL https://github.com/scrapy/scrapy/raw/master/extras/scrapy_bash_completion -o /etc/bash_completion.d/scrapy_bash_completion \ && echo 'source /etc/bash_completion.d/scrapy_bash_completion' >> /root/.bashrc \ && apt-get remove -y autoconf \ @@ -44,15 +48,11 @@ RUN apt-get update \ libssl-dev \ libtool \ python-dev \ - && apt-get autoremove -y + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* -ADD ./scrapyd.conf /etc/scrapyd/ VOLUME /etc/scrapyd/ /var/lib/scrapyd/ + EXPOSE 6800 -ONBUILD ADD ./packages.txt /etc/scrapyd/ -ONBUILD RUN cat /etc/scrapyd/packages.txt | xargs apt-get install -y -ONBUILD ADD ./requirements.txt /etc/scrapyd/ -ONBUILD RUN pip install -r /etc/scrapyd/requirements.txt - CMD ["scrapyd"] diff --git a/scrapyd/Dockerfile.old b/scrapyd.old/Dockerfile.old similarity index 100% rename from scrapyd/Dockerfile.old rename to scrapyd.old/Dockerfile.old diff --git a/scrapyd.old/README.md b/scrapyd.old/README.md new file mode 100644 index 0000000..212fa23 --- /dev/null +++ b/scrapyd.old/README.md @@ -0,0 +1,15 @@ +docker-scrapyd +============== + +Dockerfile for building an image that runs [scrapyd][1]. + +## Building + + $ docker build -t scrapyd . + +## Running + + $ docker run -p 6800:6800 scrapyd + +[1]: https://github.com/scrapy/scrapyd + diff --git a/scrapyd.old/requirements.txt b/scrapyd.old/requirements.txt new file mode 100644 index 0000000..6418dc4 --- /dev/null +++ b/scrapyd.old/requirements.txt @@ -0,0 +1,21 @@ +--allow-all-external +--allow-unverified jsonpath + +# parser +git+https://github.com/scrapy/scrapely +jsonpath +jsonschema +pyparsing +pyquery +pyyaml + +# database +pymongo +redis + +# others +chardet +requests +service_identity +toolz + diff --git a/scrapyd-onbuild/scrapyd.conf b/scrapyd.old/scrapyd.conf similarity index 100% rename from scrapyd-onbuild/scrapyd.conf rename to scrapyd.old/scrapyd.conf diff --git a/scrapyd/Dockerfile b/scrapyd/Dockerfile index 860b687..a070bf1 100644 --- a/scrapyd/Dockerfile +++ b/scrapyd/Dockerfile @@ -1,11 +1,11 @@ # -# Dockerfile for scrapyd +# Dockerfile for scrapyd-onbuild # FROM debian:jessie MAINTAINER kev -ADD ./scrapyd.conf /etc/scrapyd/ +ADD ./packages.txt /etc/scrapyd/ ADD ./requirements.txt /etc/scrapyd/ RUN apt-get update \ @@ -39,7 +39,9 @@ RUN apt-get update \ && rm -rf libxslt \ && curl -sSL https://bootstrap.pypa.io/get-pip.py | python \ && pip install scrapy==0.24.6 scrapyd==1.0.1 \ + && xargs apt-get install -y < /etc/scrapyd/packages.txt \ && pip install -r /etc/scrapyd/requirements.txt \ + && xargs apt-get remove -y < /etc/scrapyd/packages.txt \ && curl -sSL https://github.com/scrapy/scrapy/raw/master/extras/scrapy_bash_completion -o /etc/bash_completion.d/scrapy_bash_completion \ && echo 'source /etc/bash_completion.d/scrapy_bash_completion' >> /root/.bashrc \ && apt-get remove -y autoconf \ @@ -51,8 +53,8 @@ RUN apt-get update \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* +ADD ./scrapyd.conf /etc/scrapyd/ VOLUME /etc/scrapyd/ /var/lib/scrapyd/ - EXPOSE 6800 CMD ["scrapyd"] diff --git a/scrapyd/README.md b/scrapyd/README.md index 212fa23..27fb3fb 100644 --- a/scrapyd/README.md +++ b/scrapyd/README.md @@ -1,15 +1,13 @@ -docker-scrapyd -============== +scrapyd +======= Dockerfile for building an image that runs [scrapyd][1]. -## Building +Please use this image as base for your own project. - $ docker build -t scrapyd . +You may edit these files: -## Running - - $ docker run -p 6800:6800 scrapyd +- `packages.txt` - dependencies for python requirements +- `requirements.txt` - additional python packages to be installed [1]: https://github.com/scrapy/scrapyd - diff --git a/scrapyd-onbuild/packages.txt b/scrapyd/packages.txt similarity index 100% rename from scrapyd-onbuild/packages.txt rename to scrapyd/packages.txt diff --git a/scrapyd/requirements.txt b/scrapyd/requirements.txt index 6418dc4..658130b 100644 --- a/scrapyd/requirements.txt +++ b/scrapyd/requirements.txt @@ -1,21 +1 @@ ---allow-all-external ---allow-unverified jsonpath - -# parser -git+https://github.com/scrapy/scrapely -jsonpath -jsonschema -pyparsing -pyquery -pyyaml - -# database -pymongo -redis - -# others -chardet -requests -service_identity -toolz - +psycopg2