From 4513bc9f846451fe39b4d0111ec67981d4b45908 Mon Sep 17 00:00:00 2001 From: Ari Timonen Date: Mon, 6 Apr 2020 03:31:05 +0300 Subject: [PATCH] Fix env vars issues Dockerfile clean up + comments entry.sh more env vars Fix env* rbs Comment out SSL for now in nginx Doc updates --- .env.production | 2 + .env.staging | 2 + .gitignore | 3 + DEVELOPMENT.md | 22 +++---- Dockerfile | 90 ++++++++++++++++------------ INSTALL.md | 6 +- bin/script/entry.sh | 18 ++++-- config/environments/production.rb | 6 +- config/environments/staging.rb | 7 ++- docker-compose.yml | 46 +++++++------- ext/nginx.conf.d/nginx.conf.template | 35 ++++++----- 11 files changed, 134 insertions(+), 103 deletions(-) diff --git a/.env.production b/.env.production index 6e363b8..98be7ab 100644 --- a/.env.production +++ b/.env.production @@ -3,6 +3,8 @@ RACK_ENV=production RAILS_ENV=production +ASSETS_PRECOMPILE=1 + # FIXME Disable workers + cluster mode for now. PUMA_WORKERS=0 PUMA_MIN_THREADS=1 diff --git a/.env.staging b/.env.staging index 610d31b..9536294 100644 --- a/.env.staging +++ b/.env.staging @@ -3,6 +3,8 @@ RACK_ENV=staging RAILS_ENV=staging +ASSETS_PRECOMPILE=1 + APP_PORT=4999 APP_PORT_SSL=5000 PUMA_PORT=5000 diff --git a/.gitignore b/.gitignore index a01227e..5224155 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ ext/nginx.conf.d/default.conf /vendor/bundle/ .bundle +# Yarn +.yarn-integrity + # Direnv .envrc diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5adfef9..9ea7913 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -6,7 +6,7 @@ Install instructions in INSTALL.md Just run and open http://localhost:4000/ - docker-compose -f docker-compose.yml up` + docker-compose up` ## Tips @@ -35,15 +35,15 @@ FIXME, TODO, EXPLAIN, OBSOLETE Load env variables: - export $(cat .env.development | xargs) && export $(cat .env | xargs) + source script/env.sh .env .env.development Start: - docker-compose -f docker-compose.yml up -d --build` + docker-compose up -d --build` Build or rebuild: - docker-compose -f docker-compose.yml build` + docker-compose build` Debug: @@ -51,19 +51,19 @@ Debug: To get inside docker web+test containers: - docker-compose -f docker-compose.yml exec -u root web /bin/bash` - docker-compose -f docker-compose.yml exec -u web web /bin/bash` - docker-compose -f docker-compose.yml exec -u root test /bin/bash` - docker-compose -f docker-compose.yml exec -u web test /bin/bash` + docker-compose exec -u root web /bin/bash` + docker-compose exec -u web web /bin/bash` + docker-compose exec -u root test /bin/bash` + docker-compose exec -u web test /bin/bash` Restart the web container - docker-compose -f docker-compose.yml restart web` + docker-compose restart web` Run some tests: - docker-compose -f docker-compose.yml exec -u web test bundle exec rspec` - docker-compose -f docker-compose.yml exec -u web test bundle exec rspec spec/controllers/shoutmsgs_controller_spec.rb` + docker-compose exec -u web test bundle exec rspec` + docker-compose exec -u web test bundle exec rspec spec/controllers/shoutmsgs_controller_spec.rb` # Design of ENSL Application diff --git a/Dockerfile b/Dockerfile index 42d847d..21a1258 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,65 +1,77 @@ FROM ruby:2.6.5 AS ensl_development ENV RAILS_ENV development +ENV DEPLOY_PATH /var/www -RUN adduser web --home /home/web --shell /bin/bash --disabled-password --gecos "" && \ - apt-get update && apt-get -y upgrade \ - && apt-get -y install \ - libmariadb-dev libmariadb-dev-compat \ - libssl-dev \ - zlib1g-dev libreadline-dev libyaml-dev \ - libxslt1-dev libxml2-dev \ - imagemagick libmagickwand-dev \ - nodejs \ - phantomjs \ - firefox-esr +RUN \ + # Add web + adduser web --home /home/web --shell /bin/bash --disabled-password --gecos "" && \ + apt-get update && apt-get -y upgrade && \ + # Pre-dependencies + apt-get -y install curl && \ + # Yarn repo + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ + echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ + # Dependencies + apt-get -y install \ + # For MySQL/MariaDB + libmariadb-dev libmariadb-dev-compat \ + # SSL libs + libssl-dev \ + # zlib, readline and libyaml + zlib1g-dev libreadline-dev libyaml-dev \ + # For nogokiri + libxslt1-dev libxml2-dev \ + # For carrierwave/rmagick + imagemagick libmagickwand-dev \ + # For javascript gems + nodejs \ + # For assets pipeline + yarn \ + # For poltergeist + phantomjs \ + firefox-esr # Separate Gemfile ADD so that `bundle install` can be cached more effectively ADD Gemfile Gemfile.lock /var/www/ RUN gem install bundler && \ - mkdir -p /var/bundle && chown -R web:web /var/bundle && \ - chown -R web:web /var/www + mkdir -p /var/bundle && chown -R web:web /var/bundle /var/www -WORKDIR /var/www USER web +WORKDIR /var/www RUN bundle config github.https true && \ bundle config set path '/var/bundle' && \ bundle install --jobs 8 -USER root - -# ENTRYPOINT ["/bin/bash"] -# CMD ["/var/www/bin/script/entry.sh"] - -# Staging - -FROM ensl_development AS ensl_staging - -ENV RAILS_ENV staging - -USER root - -ENTRYPOINT ["/bin/bash"] -CMD ["/var/www/bin/script/entry.sh"] - +# # Production +# FROM ensl_development AS ensl_production ENV RAILS_ENV production -ADD . /var/www +ADD --chown=web . /var/www -WORKDIR /var/www +# USER root +# RUN chown -R web:web /var/www +# USER web -RUN chown -R web:web /var/www - -USER web +# Assets are only compiled for production+ RUN bundle exec rake assets:precompile && \ - # Temporary fix for assets - mv /var/www/public/assets /home/web/assets + # FIXME: Temporary fix for assets + # Move assets to a temp dir here and move them back in entry script + cp -r /var/www/public/assets /home/web/assets -ENTRYPOINT ["/bin/bash"] -CMD ["/var/www/bin/script/entry.sh"] +# +# Staging +# + +FROM ensl_production AS ensl_staging + +ENV RAILS_ENV staging + +# ENTRYPOINT ["/bin/bash"] +# CMD ["/var/www/bin/script/entry.sh"] diff --git a/INSTALL.md b/INSTALL.md index 2bbc59a..55d89dd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -25,11 +25,11 @@ Install git: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git cd ensl.org docker-compose build -## 4. First select your environment (eg. development). Then use a script to load the env vars to your shell env: +## 4. First select your environment (eg. production). Then use a script to load the env vars to your shell env: - source script/env.sh .env .env.development + source script/env.sh .env .env.production -## 5. Put any database dumps to `db/initdb.d`. +## 5. Put any database dumps to `db/initdb.d`. (optional) cp dump.sql db/initdb.d/00_dump.sql diff --git a/bin/script/entry.sh b/bin/script/entry.sh index 0f6a415..af464b8 100755 --- a/bin/script/entry.sh +++ b/bin/script/entry.sh @@ -4,12 +4,18 @@ cd /var/www source script/env.sh .env .env.$RAILS_ENV .env.$RAILS_ENV.local .env.local -if [ $RAILS_ENV = "production" ]; then - rm -rf /var/www/public/assets - mv /home/web/assets /var/www/public/ - chown -R web:web /var/www +# Make sure we have all assets +su -c "bundle config github.https true; cd $DEPLOY_PATH && bundle install --path /var/bundle --jobs 4" -s /bin/bash -l web + +if [ -z $ASSETS_PRECOMPILE ] && [ $ASSETS_PRECOMPILE -eq 1 ]; then + if [[ -z "$ASSETS_PATH" ]] && [ -d "$ASSETS_PATH"]; then + rm -rf "${DEPLOY_PATH}/public/assets" + mv "$ASSETS_PATH" "${DEPLOY_PATH}/public/assets" + else + su -c "cd $DEPLOY_PATH && bundle assets:precompile" -s /bin/bash -l web + fi + chown -R web:web $DEPLOY_PATH fi -su -c "bundle config github.https true; cd /var/www && bundle install --path /var/bundle --jobs 4" -s /bin/bash -l web -su -c "cd /var/www && bundle exec puma -C config/puma.rb" -s /bin/bash -l web +su -c "cd $DEPLOY_PATH && bundle exec puma -C config/puma.rb" -s /bin/bash -l web bash diff --git a/config/environments/production.rb b/config/environments/production.rb index e52bc84..d18732a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -13,7 +13,7 @@ Ensl::Application.configure do # Compress JavaScripts and CSS config.assets.compress = true - config.assets.js_compressor = :uglifier + # config.assets.js_compressor = :uglifier # Don't fallback to assets pipeline if a precompiled asset is missed config.assets.compile = true @@ -25,8 +25,8 @@ Ensl::Application.configure do # config.assets.manifest = YOUR_PATH # Specifies the header that your server uses for sending files - config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache - # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx + # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache + config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 7f7ee3e..b28c3bb 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -13,7 +13,7 @@ Ensl::Application.configure do # Compress JavaScripts and CSS config.assets.compress = true - config.assets.js_compressor = :uglifier + # config.assets.js_compressor = :uglifier # Don't fallback to assets pipeline if a precompiled asset is missed config.assets.compile = true @@ -25,8 +25,8 @@ Ensl::Application.configure do # config.assets.manifest = YOUR_PATH # Specifies the header that your server uses for sending files - config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache - # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx + # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache + config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true @@ -67,5 +67,6 @@ Ensl::Application.configure do # Custom Session Store config to allow gathers.staging.ensl.org config.session_store :cookie_store, key: "_ENSL_session_key_staging", expire_after: 30.days.to_i + # Load all models auto config.eager_load = true end diff --git a/docker-compose.yml b/docker-compose.yml index 671621e..883e3af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,24 @@ services: #- spring #- redis + nginx: + image: nginx:latest + container_name: ensl_${RAILS_ENV}_nginx + command: /bin/bash -c "envsubst '$$PUMA_PORT' < /etc/nginx/conf.d/nginx.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" + volumes: + - ./ext/ssl/fullchain.pem:/etc/ssl/certs/ensl_fullchain.pem + - ./ext/ssl/privkey.pem:/etc/ssl/private/ensl_privkey.pem + - ./ext/nginx.conf.d/:/etc/nginx/conf.d/ + - ./public:/var/www/public + #ports: + # - $APP_PORT:80 + # - $APP_PORT_SSL:443 + environment: + - APP_DOMAIN=$APP_DOMAIN + - APP_PORT=$APP_PORT + - PUMA_PORT=$PUMA_PORT + - RAILS_ENV=$RAILS_ENV + # # Testing # @@ -62,7 +80,7 @@ services: selenium: image: selenium/standalone-chrome-debug - container_name: ensl_${RAILS_ENV}_selenium + container_name: ensl_selenium ports: - 5900:5900 - 4444:4444 @@ -73,15 +91,15 @@ services: memcached: image: memcached:alpine - container_name: ensl_${RAILS_ENV}_memcached + container_name: ensl_memcached redis: image: 'redis:4.0-alpine' - container_name: ensl_${RAILS_ENV}_redis + container_name: ensl_redis db: image: mariadb:latest - container_name: ensl_${RAILS_ENV}_db + container_name: ensl_db user: "mysql:mysql" # debug; command: mysqld_safe --skip-grant-tables volumes: @@ -97,7 +115,7 @@ services: smtp: image: mwader/postfix-relay:latest - container_name: ensl_${RAILS_ENV}_smtp + container_name: ensl_smtp restart: always volumes: - "./ext/dkim:/etc/opendkim/keys" @@ -105,24 +123,6 @@ services: - POSTFIX_myhostname=$APP_DOMAIN - OPENDKIM_DOMAINS=$APP_DOMAIN - nginx: - image: nginx:latest - container_name: ensl_${RAILS_ENV}_nginx - command: /bin/bash -c "envsubst '$$PUMA_PORT' < /etc/nginx/conf.d/nginx.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" - volumes: - - ./ext/ssl/fullchain.pem:/etc/ssl/certs/ensl_fullchain.pem - - ./ext/ssl/privkey.pem:/etc/ssl/private/ensl_privkey.pem - - ./ext/nginx.conf.d/:/etc/nginx/conf.d/ - - ./public:/var/www - ports: - - $APP_PORT:80 - - $APP_PORT_SSL:443 - environment: - - APP_DOMAIN=$APP_DOMAIN - - APP_PORT=$APP_PORT - - PUMA_PORT=$PUMA_PORT - - RAILS_ENV=$RAILS_ENV - # spring: # build: # context: ./ diff --git a/ext/nginx.conf.d/nginx.conf.template b/ext/nginx.conf.d/nginx.conf.template index 3fea65b..68ad71b 100644 --- a/ext/nginx.conf.d/nginx.conf.template +++ b/ext/nginx.conf.d/nginx.conf.template @@ -12,37 +12,42 @@ server { listen *:80; listen *:443 ssl; - ssl_certificate /etc/ssl/certs/ensl_fullchain.pem; - ssl_certificate_key /etc/ssl/private/ensl_privkey.pem; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + # ssl_certificate /etc/ssl/certs/ensl_fullchain.pem; + # ssl_certificate_key /etc/ssl/private/ensl_privkey.pem; + # ssl_protocols TLSv1 TLSv1.1 TLSv1.2; server_name ensl.org; - root /var/www; + root /var/www/public; return 301 https://www.ensl.org$request_uri; } # HTTP -> HTTPS redirect -server { - listen *:80; - server_name www.ensl.org; - return 301 https://www.ensl.org$request_uri; -} +#server { +# listen *:80; +# server_name www.ensl.org; +# return 301 https://www.ensl.org$request_uri; +#} server { - listen *:443 ssl default_server; - server_name www.ensl.org; - root /var/www; + # listen *:443 ssl default_server; + listen *:80 default_server; + # server_name www.ensl.org; + root /var/www/public; index index.html index.htm index.php; - ssl_certificate /etc/ssl/certs/ensl_fullchain.pem; - ssl_certificate_key /etc/ssl/private/ensl_privkey.pem; + # ssl_certificate /etc/ssl/certs/ensl_fullchain.pem; + # ssl_certificate_key /etc/ssl/private/ensl_privkey.pem; + + # ssl-cert /etc/ssl/certs/ssl-cert-snakeoil.pem + # ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA; ssl_session_timeout 1d; ssl_session_cache shared:SSL:50m; ssl_stapling on; ssl_stapling_verify on; - #add_header Strict-Transport-Security max-age=15768000; + # add_header Strict-Transport-Security max-age=15768000; access_log /var/log/nginx/ensl.access.log; error_log /var/log/nginx/ensl.error.log;