From 30b9198b6df86e0cc27b23b74c30499af5d4b5f2 Mon Sep 17 00:00:00 2001 From: Ari Timonen Date: Wed, 8 Apr 2020 22:25:24 +0300 Subject: [PATCH] Add staging that works Tune env vars Fix logs in gitignore Rename DEPLOY_PATH env var --- .dockerignore | 2 +- .env | 31 +++--- .env.development | 6 +- .env.production | 2 +- .env.staging | 8 +- .env.test | 2 +- .gitignore | 2 +- DEVELOPMENT.md | 68 ++++++------ Dockerfile | 14 +-- INSTALL.md | 17 +-- bin/script/entry.sh | 25 +++-- config/puma.rb | 6 +- db/initdb.d/99_ensl.setup.sql | 3 + docker-compose.yml | 101 +++++++++++++----- ...nx.conf.template => staging.conf.template} | 39 ++++--- script/env.sh | 3 +- 16 files changed, 204 insertions(+), 125 deletions(-) rename ext/nginx.conf.d/{nginx.conf.template => staging.conf.template} (78%) diff --git a/.dockerignore b/.dockerignore index d9ca2ff..bf4b398 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,7 @@ docker-compose.yml *.md .idea dkim -log/.log* +log/*.log* tmp/ public/* db_data diff --git a/.env b/.env index dec7243..ce6a8a5 100644 --- a/.env +++ b/.env @@ -5,24 +5,31 @@ RACK_ENV=production RAILS_ENV=production -# App domain, used mostly bt just postfix -# The app is designed as domain-indepdendent -APP_DOMAIN=ensl.org - # App secret for cookie encryption, blank is random APP_SECRET= -# Public ports -APP_PORT=80 -APP_PORT_SSL=443 +# FIXME: This doesn't really belong here +# Find a better solution. Since nginx needs all these variables for envsubst +# and you don't really want to start multiple nginx instances. +PRODUCTION_PORT=80 +PRODUCTION_PORT_SSL=443 +PRODUCTION_DOMAIN=www.ensl.org +STAGING_PORT=5001 +STAGING_PORT_SSL=5000 +STAGING_DOMAIN=www.ensl.org +STAGING_ROOT_DOMAIN=ensl.org -# FIXME: doesn't work yet +# FIXME: SCRYPT_SALT_OPTS doesn't work yet # Options for: SCrypt::Engine.calibrate!(max_mem: 16 * 1024 * 1024) SCRYPT_SALT_OPTS= -SCRYPT_MAX_TIME=1 +SCRYPT_MAX_TIME=0.01 + +# SMTP server +MAIL_DOMAIN=ensl.org # App path inside docker -DEPLOY_PATH=/var/www +APP_PATH=/var/www +APP_PATH_PUBLIC=/var/www/public ASSETS_PATH=/home/web/assets ASSETS_PRECOMPILE=0 @@ -57,9 +64,9 @@ MYSQL_ROOT_HOST=% # More MySQL vars MYSQL_CONNECTION_POOL=32 -# Test +# Test settings SELENIUM_HOST=selenium -TEST_APP_HOST=localhost +TEST_APP_HOST=test TEST_APP_PORT=3005 # New relic diff --git a/.env.development b/.env.development index 111169e..a89afc0 100644 --- a/.env.development +++ b/.env.development @@ -3,8 +3,10 @@ # Development-specific ENV variables, loaded after .env and before .env*local # Add changes only specific to dev. env. -# Scrypt max time -SCRYPT_MAX_TIME=0.005 +# Scrypt max time to reduce migration time +SCRYPT_MAX_TIME=0.001 + +MYSQL_DATABASE=ensl_development # These are RACK_ENV=development diff --git a/.env.production b/.env.production index 5adfd2f..e65f73a 100644 --- a/.env.production +++ b/.env.production @@ -5,7 +5,7 @@ RAILS_ENV=production ASSETS_PRECOMPILE=1 -SCRYPT_MAX_TIME=0.5 +SCRYPT_MAX_TIME=0.001 # FIXME Disable workers + cluster mode for now. PUMA_WORKERS=0 diff --git a/.env.staging b/.env.staging index 9536294..d9517f7 100644 --- a/.env.staging +++ b/.env.staging @@ -3,10 +3,14 @@ RACK_ENV=staging RAILS_ENV=staging +SCRYPT_MAX_TIME=0.001 + ASSETS_PRECOMPILE=1 -APP_PORT=4999 -APP_PORT_SSL=5000 PUMA_PORT=5000 +STAGING_ROOT_DOMAIN=ensl.org +STAGING_DOMAIN=www.ensl.org +STAGING_PORT=5001 +STAGING_PORT_SSL=5000 MYSQL_DATABASE=ensl_staging \ No newline at end of file diff --git a/.env.test b/.env.test index 0c74f9a..a467c03 100644 --- a/.env.test +++ b/.env.test @@ -5,7 +5,7 @@ RAILS_ENV=test APP_SECRET=fe837ea72667ec3d8ecb94cfba1a1bba -DEPLOY_PATH=/var/www +APP_PATH=/var/www PUMA_PORT=4000 PUMA_TIMEOUT=30 diff --git a/.gitignore b/.gitignore index aa12cb1..50470c6 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,7 @@ ext/dkim ext/ssl !ext/ssl/.placeholder -# Nginx +# Nginx; only the default cfg is ignored. ext/nginx.conf.d/default.conf # Uploaded Assets diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9ea7913..f777c94 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,36 +2,7 @@ Install instructions in INSTALL.md -## Startup - -Just run and open http://localhost:4000/ - - docker-compose up` - -## Tips - -1. Everything should be running on containers. -1. If you need to run stuff on your host (eg. ruby, rubocop, bundle install etc) run all commands from the: `Dockerfile.dev`. It should setup identical setup for your machine. -1. Add docker container names to /etc/hosts. This makes it possible to run test from local machine without using the container since editor/IDE don't integrate with Docker so well. - - sudo echo `docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ensl_dev_db` db >> /etc/hosts - -1. VS Code and RubyMine are great IDE's/editors. -1. To run VS Code plugin Ruby Test Explorer in docker container you need to create path to custom path, copy the formatter and it whines about -and it still fails a bit. https://github.com/connorshea/vscode-ruby-test-adapter/issues/21 -1. Do not commit too much without testing. Also keep commits small for documentation and reversability issues. -1. You need to rebuild the docker image when you change gems. - -## TODO issues for dev - -1. Puma should be running (eg. spring), and if debugger is used it should be able to connect via docker-compose up -1. Should directories exist? - -# Tags in code - -FIXME, TODO, EXPLAIN, OBSOLETE - -## Handy commands +## Bassic commands for development Load env variables: @@ -39,7 +10,7 @@ Load env variables: Start: - docker-compose up -d --build` + docker-compose up --build development` Build or rebuild: @@ -47,7 +18,7 @@ Build or rebuild: Debug: - docker attach ensl_dev + docker attach ensl_development To get inside docker web+test containers: @@ -65,17 +36,44 @@ Run some tests: docker-compose exec -u web test bundle exec rspec` docker-compose exec -u web test bundle exec rspec spec/controllers/shoutmsgs_controller_spec.rb` -# Design of ENSL Application +## Tips + +1. If you need to run stuff on your host (eg. ruby, rubocop, bundle install etc) run all commands from the: `Dockerfile.dev`. It should setup identical setup for your machine. +1. Add docker container names to /etc/hosts. This makes it possible to run test from local machine without using the container since editor/IDE don't integrate with Docker so well. + + sudo echo `docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ensl_dev_db` db >> /etc/hosts + +1. VS Code and RubyMine are great IDE's/editors. +1. To run VS Code plugin Ruby Test Explorer in docker container you need to create path to custom path, copy the formatter and it whines about and it [still fails a bit](https://github.com/connorshea/vscode-ruby-test-adapter/issues/21). +1. Do not commit too much without testing. Also keep commits small for documentation and reversability issues. +1. You need to rebuild the docker image when you change gems. + +## Design of ENSL Application Read this to understand design decisions and follow them! 1. Env variables should be used everywhere and loaded from .env* files using Dotenv -1. The app contents are added to the docker image on build but it is mounted as **volume**. -1. Use rails / ruby best practices in section below. +1. Everything should be running on containers. +1. Docker-compose is the heart of deployment +1. Dockerfile should contain the gems and prebuilt assets for production +1. The app contents are added to the docker image *on build* but it is mounted as **volume**. It will override the Dockerfile content. + +## Tags in code + +FIXME, TODO, EXPLAIN, OBSOLETE + +## TODO issues for dev + +1. Puma should be running (eg. spring), and if debugger is used it should be able to connect via docker-compose up +1. Should directories exist? +1. docker-compose has some .env specific vars and then ## Best practices +Take a look at these before writing anything. + 1. https://nvie.com/posts/a-successful-git-branching-model/ +1. https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ 1. https://github.com/rubocop-hq/ruby-style-guide 1. https://rails-bestpractices.com/ 1. http://www.betterspecs.org/ diff --git a/Dockerfile b/Dockerfile index 21a1258..1cde815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ruby:2.6.5 AS ensl_development ENV RAILS_ENV development -ENV DEPLOY_PATH /var/www +ENV APP_PATH /var/www RUN \ # Add web @@ -13,7 +13,7 @@ RUN \ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ # Dependencies - apt-get -y install \ + apt-get -y install --upgrade \ # For MySQL/MariaDB libmariadb-dev libmariadb-dev-compat \ # SSL libs @@ -30,13 +30,13 @@ RUN \ yarn \ # For poltergeist phantomjs \ - firefox-esr + firefox-esr && \ + # Install bundler and bundle path + gem install bundler && \ + mkdir -p /var/bundle && chown -R web:web /var/bundle # Separate Gemfile ADD so that `bundle install` can be cached more effectively -ADD Gemfile Gemfile.lock /var/www/ - -RUN gem install bundler && \ - mkdir -p /var/bundle && chown -R web:web /var/bundle /var/www +ADD --chown=web Gemfile Gemfile.lock /var/www/ USER web WORKDIR /var/www diff --git a/INSTALL.md b/INSTALL.md index 55d89dd..1fa44ae 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -25,24 +25,25 @@ Install git: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git cd ensl.org docker-compose build -## 4. First select your environment (eg. production). Then use a script to load the env vars to your shell env: +## 4. Put any database dumps to `db/initdb.d`. (optional) - source script/env.sh .env .env.production - -## 5. Put any database dumps to `db/initdb.d`. (optional) - - cp dump.sql db/initdb.d/00_dump.sql + mysqldump --opt -h DATABASE_IP -u USERNAME DATABASE_NAME > 00_ensl.org.`date +%F`.sql + mv 00_ensl.org.`date +%F`.sql db/initdb.d/00_ensl.org.`date +%F`.sql You need to manually copy it to staging database on same db server for now. +## 5. First select your environment (eg. production). Then use a script to load the env vars to your shell env: + + source script/env.sh .env .env.production + ## 6. Then start the whole thing - docker-compose up + docker-compose up production docker-compose down ## 7. Install reverse proxy (production only) -a) The docker-compose contains basic nginx setup. Use that. +a) The docker-compose contains basic nginx setup. It's in docker-compose. Use that. b) If you have your own NGINX setup, just use the sample site file from the ext/nginx.conf.d diff --git a/bin/script/entry.sh b/bin/script/entry.sh index af464b8..5ba01bc 100755 --- a/bin/script/entry.sh +++ b/bin/script/entry.sh @@ -1,21 +1,28 @@ #!/bin/bash -cd /var/www +cd $APP_PATH source script/env.sh .env .env.$RAILS_ENV .env.$RAILS_ENV.local .env.local # Make sure we have all assets -su -c "bundle config github.https true; cd $DEPLOY_PATH && bundle install --path /var/bundle --jobs 4" -s /bin/bash -l web +bundle config github.https true +bundle config set path '/var/bundle' +bundle install --jobs 8 -if [ -z $ASSETS_PRECOMPILE ] && [ $ASSETS_PRECOMPILE -eq 1 ]; then - if [[ -z "$ASSETS_PATH" ]] && [ -d "$ASSETS_PATH"]; then - rm -rf "${DEPLOY_PATH}/public/assets" - mv "$ASSETS_PATH" "${DEPLOY_PATH}/public/assets" +if [ "$ASSETS_PRECOMPILE" -eq 1 ]; then + echo "Fetching assets..." + if false; then + #if [[ -z "$ASSETS_PATH" ]] && [ -d "$ASSETS_PATH"]; then + rm -rf "${APP_PATH}/public/assets" + mv "$ASSETS_PATH" "${APP_PATH}/public/assets" else - su -c "cd $DEPLOY_PATH && bundle assets:precompile" -s /bin/bash -l web + cd $APP_PATH + bundle exec rake assets:clean + bundle exec rake assets:precompile fi - chown -R web:web $DEPLOY_PATH + chown -R web:web $APP_PATH fi -su -c "cd $DEPLOY_PATH && bundle exec puma -C config/puma.rb" -s /bin/bash -l web +cd $APP_PATH +bundle exec puma -C config/puma.rb bash diff --git a/config/puma.rb b/config/puma.rb index eb48465..618296e 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -15,11 +15,11 @@ rackup DefaultRackup # Set vars as we cannmot load them rails_env = ENV['RAILS_ENV'] || 'development' -app_dir = ENV['DEPLOY_PATH'] || '/var/www' +app_dir = ENV['APP_PATH'] || '/var/www' # Set basic puma settings environment rails_env -bind "unix://#{app_dir}/tmp/sockets/puma.sock" +bind "unix://#{app_dir}/tmp/sockets/puma.#{rails_env}.sock" port Integer(ENV['PUMA_PORT'] || 4000) # Redirect stdout only in production. Dev mode needs it for debug @@ -50,6 +50,6 @@ end # EXPLAIN This has been added here but why? on_restart do ENV["BUNDLE_GEMFILE"] = "#{app_dir}/Gemfile" - Dotenv.overload + Dotenv.overload('.env.' + ENV['RAILS_ENV'] + '.local', '.env.local', '.env.' + ENV['RAILS_ENV'], '.env') ActiveRecord::Base.connection.disconnect! end diff --git a/db/initdb.d/99_ensl.setup.sql b/db/initdb.d/99_ensl.setup.sql index edcbdb0..fcbbaaa 100644 --- a/db/initdb.d/99_ensl.setup.sql +++ b/db/initdb.d/99_ensl.setup.sql @@ -1,4 +1,7 @@ -- FIXME: this should be somewhere else probably +CREATE DATABASE ensl_development; +GRANT ALL PRIVILEGES ON ensl_development.* TO 'ensl'@'%' WITH GRANT OPTION; + CREATE DATABASE ensl_test; GRANT ALL PRIVILEGES ON ensl_test.* TO 'ensl'@'%' WITH GRANT OPTION; diff --git a/docker-compose.yml b/docker-compose.yml index 883e3af..2f31766 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,15 +2,61 @@ version: "3.4" services: # - # Main service + # Main services # - web: + production: + command: "bundle exec puma" + tty: true + stdin_open: true + container_name: "ensl_production" + user: "web:web" + build: + context: ./ + dockerfile: Dockerfile + target: ensl_production + volumes: + - ".:/var/www/" + environment: + RAILS_ENV: production + ports: + - "${PUMA_PORT}:${PUMA_PORT}" + # Guard livereload + # - "35729:35729" + links: + - db + - memcached + - smtp + + staging: + # command: "/bin/bash" + # command: "/var/www/bin/script/entry.sh" + command: "bundle exec puma" + tty: true + stdin_open: true + container_name: "ensl_staging" + user: "web:web" + build: + context: ./ + dockerfile: Dockerfile + target: ensl_staging + volumes: + - ".:/var/www/" + environment: + RAILS_ENV: staging + ports: + - "9000:${PUMA_PORT}" + links: + - db + - memcached + - smtp + + development: # For debug; Use attach rather than this # command: /bin/bash command: "bundle exec puma" tty: true stdin_open: true - container_name: "ensl_${RAILS_ENV}" + container_name: "ensl_development" user: "web:web" build: context: ./ @@ -26,27 +72,6 @@ services: links: - db - memcached - - smtp - #- spring - #- redis - - nginx: - image: nginx:latest - container_name: ensl_${RAILS_ENV}_nginx - command: /bin/bash -c "envsubst '$$PUMA_PORT' < /etc/nginx/conf.d/nginx.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" - volumes: - - ./ext/ssl/fullchain.pem:/etc/ssl/certs/ensl_fullchain.pem - - ./ext/ssl/privkey.pem:/etc/ssl/private/ensl_privkey.pem - - ./ext/nginx.conf.d/:/etc/nginx/conf.d/ - - ./public:/var/www/public - #ports: - # - $APP_PORT:80 - # - $APP_PORT_SSL:443 - environment: - - APP_DOMAIN=$APP_DOMAIN - - APP_PORT=$APP_PORT - - PUMA_PORT=$PUMA_PORT - - RAILS_ENV=$RAILS_ENV # # Testing @@ -120,8 +145,32 @@ services: volumes: - "./ext/dkim:/etc/opendkim/keys" environment: - - POSTFIX_myhostname=$APP_DOMAIN - - OPENDKIM_DOMAINS=$APP_DOMAIN + - POSTFIX_myhostname=$MAIL_DOMAIN + - OPENDKIM_DOMAINS=$MAIL_DOMAIN + + nginx: + image: nginx:latest + container_name: ensl_nginx + tty: true + stdin_open: true + command: /bin/bash -c "envsubst '$$PUMA_PORT $$APP_PATH $$APP_PATH_PUBLIC $$STAGING_ROOT_DOMAIN $$STAGING_DOMAIN $$STAGING_PORT $$STAGING_PORT_SSL' < /etc/nginx/conf.d/*.conf.template > /etc/nginx/conf.d/default.conf && cat /etc/nginx/conf.d/*.conf && nginx -g 'daemon off;'" + volumes: + - ./ext/ssl/fullchain.pem:/etc/ssl/certs/ensl_fullchain.pem + - ./ext/ssl/privkey.pem:/etc/ssl/private/ensl_privkey.pem + - ./ext/nginx.conf.d/:/etc/nginx/conf.d/ + - ./public:/var/www/public + ports: + - $STAGING_PORT:$STAGING_PORT + - $STAGING_PORT_SSL:$STAGING_PORT_SSL + environment: + - PUMA_PORT=$PUMA_PORT + - STAGING_ROOT_DOMAIN=$STAGING_ROOT_DOMAIN + - STAGING_DOMAIN=$STAGING_DOMAIN + - STAGING_PORT=$STAGING_PORT + - STAGING_PORT_SSL=$STAGING_PORT_SSL + - RAILS_ENV=$RAILS_ENV + - APP_PATH=$APP_PATH + - APP_PATH_PUBLIC=$APP_PATH_PUBLIC # spring: # build: diff --git a/ext/nginx.conf.d/nginx.conf.template b/ext/nginx.conf.d/staging.conf.template similarity index 78% rename from ext/nginx.conf.d/nginx.conf.template rename to ext/nginx.conf.d/staging.conf.template index 88789f6..ea89327 100644 --- a/ext/nginx.conf.d/nginx.conf.template +++ b/ext/nginx.conf.d/staging.conf.template @@ -3,36 +3,39 @@ # Use it in production or copy it over upstream puma { - server localhost:$PUMA_PORT; + server staging:$PUMA_PORT; # server unix:/var/tmp/puma.$RAILS_ENV.sock fail_timeout=0; } # root-level -> www redirect -server { - listen *:80; - listen *:443 ssl; +#server { +# listen *:$STAGING_PORT; +# listen *:$STAGING_PORT_SSL ssl; + # FIXME # ssl_certificate /etc/ssl/certs/ensl_fullchain.pem; # ssl_certificate_key /etc/ssl/private/ensl_privkey.pem; # ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - server_name ensl.org; - root /var/www/public; - return 301 https://www.ensl.org$request_uri; -} +# #server_name $STAGING_ROOT_DOMAIN; +# #root $APP_PATH_PUBLIC; +# return 301 https://$STAGING_DOMAIN$request_uri; +#} # HTTP -> HTTPS redirect #server { -# listen *:80; -# server_name www.ensl.org; -# return 301 https://www.ensl.org$request_uri; +# listen *:STAGING_PORT; +# server_name $STAGING_DOMAIN; +# return 301 https://$STAGING_DOMAIN$request_uri; #} server { - # listen *:443 ssl default_server; - listen *:80 default_server; - # server_name www.ensl.org; - root /var/www/public; + listen *:$STAGING_PORT default_server; + # SSL disabled for now + # listen *:STAGING_PORT_SSL ssl default_server; + # domain_agnostic staging + # server_name $STAGING_DOMAIN; + root $APP_PATH_PUBLIC; index index.html index.htm index.php; # ssl_certificate /etc/ssl/certs/ensl_fullchain.pem; @@ -60,14 +63,18 @@ server { allow all; autoindex on; } + + # FIXME: use env. var location ^~ /assets/ { gzip_static on; expires max; add_header Cache-Control public; } + + # FIXME: use env. var location /files/ { # try_files $uri $uri/ @puma; - # alias root /var/www/public/files/; + # alias root $APP_PATH_PUBLIC/files/; autoindex on; } location @puma { diff --git a/script/env.sh b/script/env.sh index b3667a7..06bb64c 100755 --- a/script/env.sh +++ b/script/env.sh @@ -13,6 +13,7 @@ do ARGS=$(cat $FILE |grep -vE '^[[:space:]]*(#.*)*$') export $(echo $ARGS|xargs) - echo "$ARGS\n" + echo "$ARGS" + echo done