Add staging that works

Tune env vars
Fix logs in gitignore
Rename DEPLOY_PATH env var

.dockerignore

@@ -3,7 +3,7 @@ docker-compose.yml
 *.md
 .idea
 dkim
-log/.log*
+log/*.log*
 tmp/
 public/*
 db_data

.env

@@ -5,24 +5,31 @@
 RACK_ENV=production
 RAILS_ENV=production
 
-# App domain, used mostly bt just postfix
-# The app is designed as domain-indepdendent
-APP_DOMAIN=ensl.org
-
 # App secret for cookie encryption, blank is random
 APP_SECRET=
 
-# Public ports
+# FIXME: This doesn't really belong here
-APP_PORT=80
+# Find a better solution. Since nginx needs all these variables for envsubst
-APP_PORT_SSL=443
+# and you don't really want to start multiple nginx instances.
+PRODUCTION_PORT=80
+PRODUCTION_PORT_SSL=443
+PRODUCTION_DOMAIN=www.ensl.org
+STAGING_PORT=5001
+STAGING_PORT_SSL=5000
+STAGING_DOMAIN=www.ensl.org
+STAGING_ROOT_DOMAIN=ensl.org
 
-# FIXME: doesn't work yet
+# FIXME: SCRYPT_SALT_OPTS doesn't work yet
 # Options for: SCrypt::Engine.calibrate!(max_mem: 16 * 1024 * 1024)
 SCRYPT_SALT_OPTS=
-SCRYPT_MAX_TIME=1
+SCRYPT_MAX_TIME=0.01
+
+# SMTP server
+MAIL_DOMAIN=ensl.org
 
 # App path inside docker
-DEPLOY_PATH=/var/www
+APP_PATH=/var/www
+APP_PATH_PUBLIC=/var/www/public
 ASSETS_PATH=/home/web/assets
 ASSETS_PRECOMPILE=0
 
@@ -57,9 +64,9 @@ MYSQL_ROOT_HOST=%
 # More MySQL vars
 MYSQL_CONNECTION_POOL=32
 
-# Test
+# Test settings
 SELENIUM_HOST=selenium
-TEST_APP_HOST=localhost
+TEST_APP_HOST=test
 TEST_APP_PORT=3005
 
 # New relic

.env.development

@@ -3,8 +3,10 @@
 # Development-specific ENV variables, loaded after .env and before .env*local
 # Add changes only specific to dev. env.
 
-# Scrypt max time
+# Scrypt max time to reduce migration time
-SCRYPT_MAX_TIME=0.005
+SCRYPT_MAX_TIME=0.001
+
+MYSQL_DATABASE=ensl_development
 
 # These are
 RACK_ENV=development

.env.production

@@ -5,7 +5,7 @@ RAILS_ENV=production
 
 ASSETS_PRECOMPILE=1
 
-SCRYPT_MAX_TIME=0.5
+SCRYPT_MAX_TIME=0.001
 
 # FIXME Disable workers + cluster mode for now.
 PUMA_WORKERS=0

.env.staging

@@ -3,10 +3,14 @@
 RACK_ENV=staging
 RAILS_ENV=staging
 
+SCRYPT_MAX_TIME=0.001
+
 ASSETS_PRECOMPILE=1
 
-APP_PORT=4999
-APP_PORT_SSL=5000
 PUMA_PORT=5000
+STAGING_ROOT_DOMAIN=ensl.org
+STAGING_DOMAIN=www.ensl.org
+STAGING_PORT=5001
+STAGING_PORT_SSL=5000
 
 MYSQL_DATABASE=ensl_staging

.env.test

@@ -5,7 +5,7 @@ RAILS_ENV=test
 
 APP_SECRET=fe837ea72667ec3d8ecb94cfba1a1bba
 
-DEPLOY_PATH=/var/www
+APP_PATH=/var/www
 
 PUMA_PORT=4000
 PUMA_TIMEOUT=30

.gitignore

@@ -34,7 +34,7 @@ ext/dkim
 ext/ssl
 !ext/ssl/.placeholder
 
-# Nginx
+# Nginx; only the default cfg is ignored.
 ext/nginx.conf.d/default.conf
 
 # Uploaded Assets

DEVELOPMENT.md

@@ -2,36 +2,7 @@
 
 Install instructions in INSTALL.md
 
-## Startup
+## Bassic commands for development
-
-Just run and open http://localhost:4000/
-
-    docker-compose up`
-
-## Tips
-
-1. Everything should be running on containers.
-1. If you need to run stuff on your host (eg. ruby, rubocop, bundle install etc) run all commands from the: `Dockerfile.dev`. It should setup identical setup for your machine.
-1. Add docker container names to /etc/hosts. This makes it possible to run test from local machine without using the container since editor/IDE don't integrate with Docker so well.
-
-    sudo echo `docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ensl_dev_db` db >> /etc/hosts
-
-1. VS Code and RubyMine are great IDE's/editors.
-1. To run VS Code plugin Ruby Test Explorer in docker container you need to create path to custom path, copy the formatter and it whines about
-and it still fails a bit. https://github.com/connorshea/vscode-ruby-test-adapter/issues/21
-1. Do not commit too much without testing. Also keep commits small for documentation and reversability issues.
-1. You need to rebuild the docker image when you change gems.
-
-## TODO issues for dev
-
-1. Puma should be running (eg. spring), and if debugger is used it should be able to connect via docker-compose up
-1. Should directories exist?
-
-# Tags in code
-
-FIXME, TODO, EXPLAIN, OBSOLETE
-
-## Handy commands
 
 Load env variables:
 
@@ -39,7 +10,7 @@ Load env variables:
 
 Start:
 
-    docker-compose up -d --build`
+    docker-compose up --build development`
 
 Build or rebuild:
 
@@ -47,7 +18,7 @@ Build or rebuild:
 
 Debug:
 
-    docker attach ensl_dev
+    docker attach ensl_development
 
 To get inside docker web+test containers:
 
@@ -65,17 +36,44 @@ Run some tests:
     docker-compose exec -u web test bundle exec rspec`
     docker-compose exec -u web test bundle exec rspec spec/controllers/shoutmsgs_controller_spec.rb`
 
-# Design of ENSL Application
+## Tips
+
+1. If you need to run stuff on your host (eg. ruby, rubocop, bundle install etc) run all commands from the: `Dockerfile.dev`. It should setup identical setup for your machine.
+1. Add docker container names to /etc/hosts. This makes it possible to run test from local machine without using the container since editor/IDE don't integrate with Docker so well.
+
+    sudo echo `docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ensl_dev_db` db >> /etc/hosts
+
+1. VS Code and RubyMine are great IDE's/editors.
+1. To run VS Code plugin Ruby Test Explorer in docker container you need to create path to custom path, copy the formatter and it whines about and it [still fails a bit](https://github.com/connorshea/vscode-ruby-test-adapter/issues/21).
+1. Do not commit too much without testing. Also keep commits small for documentation and reversability issues.
+1. You need to rebuild the docker image when you change gems.
+
+## Design of ENSL Application
 
 Read this to understand design decisions and follow them!
 
 1. Env variables should be used everywhere and loaded from .env* files using Dotenv
-1. The app contents are added to the docker image on build but it is mounted as **volume**.
+1. Everything should be running on containers.
-1. Use rails / ruby best practices in section below.
+1. Docker-compose is the heart of deployment
+1. Dockerfile should contain the gems and prebuilt assets for production
+1. The app contents are added to the docker image *on build* but it is mounted as **volume**. It will override the Dockerfile content.
+
+## Tags in code
+
+FIXME, TODO, EXPLAIN, OBSOLETE
+
+## TODO issues for dev
+
+1. Puma should be running (eg. spring), and if debugger is used it should be able to connect via docker-compose up
+1. Should directories exist?
+1. docker-compose has some .env specific vars and then
 
 ## Best practices
 
+Take a look at these before writing anything.
+
 1. https://nvie.com/posts/a-successful-git-branching-model/
+1. https://docs.docker.com/develop/develop-images/dockerfile_best-practices/
 1. https://github.com/rubocop-hq/ruby-style-guide
 1. https://rails-bestpractices.com/
 1. http://www.betterspecs.org/

Dockerfile

@@ -1,7 +1,7 @@
 FROM ruby:2.6.5 AS ensl_development
 
 ENV RAILS_ENV development
-ENV DEPLOY_PATH /var/www
+ENV APP_PATH /var/www
 
 RUN \
     # Add web
@@ -13,7 +13,7 @@ RUN \
     curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
     echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
     # Dependencies
-    apt-get -y install \
+    apt-get -y install --upgrade \
       # For MySQL/MariaDB
       libmariadb-dev libmariadb-dev-compat \
       # SSL libs
@@ -30,13 +30,13 @@ RUN \
       yarn \
       # For poltergeist
       phantomjs \
-      firefox-esr
+      firefox-esr && \
+    # Install bundler and bundle path
+    gem install bundler && \
+    mkdir -p /var/bundle && chown -R web:web /var/bundle
 
 # Separate Gemfile ADD so that `bundle install` can be cached more effectively
-ADD Gemfile Gemfile.lock /var/www/
+ADD --chown=web Gemfile Gemfile.lock /var/www/
-
-RUN gem install bundler && \
-    mkdir -p /var/bundle && chown -R web:web /var/bundle /var/www
 
 USER web
 WORKDIR /var/www

INSTALL.md

@@ -25,24 +25,25 @@ Install git: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git
     cd ensl.org
     docker-compose build
 
-## 4. First select your environment (eg. production). Then use a script to load the env vars to your shell env:
+## 4. Put any database dumps to `db/initdb.d`. (optional)
 
-    source script/env.sh .env .env.production
+    mysqldump --opt -h DATABASE_IP -u USERNAME DATABASE_NAME > 00_ensl.org.`date +%F`.sql
-
+    mv 00_ensl.org.`date +%F`.sql db/initdb.d/00_ensl.org.`date +%F`.sql
-## 5. Put any database dumps to `db/initdb.d`. (optional)
-
-    cp dump.sql db/initdb.d/00_dump.sql
 
 You need to manually copy it to staging database on same db server for now.
 
+## 5. First select your environment (eg. production). Then use a script to load the env vars to your shell env:
+
+    source script/env.sh .env .env.production
+
 ## 6. Then start the whole thing
 
-    docker-compose up
+    docker-compose up production
     docker-compose down
 
 ## 7. Install reverse proxy (production only)
 
-a) The docker-compose contains basic nginx setup. Use that.
+a) The docker-compose contains basic nginx setup. It's in docker-compose. Use that.
 
 b) If you have your own NGINX setup, just use the sample site file from the ext/nginx.conf.d
 

bin/script/entry.sh

@@ -1,21 +1,28 @@
 #!/bin/bash
 
-cd /var/www
+cd $APP_PATH
 
 source script/env.sh .env .env.$RAILS_ENV .env.$RAILS_ENV.local .env.local
 
 # Make sure we have all assets
-su -c "bundle config github.https true; cd $DEPLOY_PATH && bundle install --path /var/bundle --jobs 4" -s /bin/bash -l web
+bundle config github.https true
+bundle config set path '/var/bundle'
+bundle install --jobs 8
 
-if [ -z $ASSETS_PRECOMPILE ] && [ $ASSETS_PRECOMPILE -eq 1 ]; then
+if [ "$ASSETS_PRECOMPILE" -eq 1 ]; then
-  if [[ -z "$ASSETS_PATH" ]] && [ -d "$ASSETS_PATH"]; then
+  echo "Fetching assets..."
-    rm -rf "${DEPLOY_PATH}/public/assets"
+  if false; then
-    mv "$ASSETS_PATH" "${DEPLOY_PATH}/public/assets"
+  #if [[ -z "$ASSETS_PATH" ]] && [ -d "$ASSETS_PATH"]; then
+    rm -rf "${APP_PATH}/public/assets"
+    mv "$ASSETS_PATH" "${APP_PATH}/public/assets"
   else
-    su -c "cd $DEPLOY_PATH && bundle assets:precompile" -s /bin/bash -l web
+    cd $APP_PATH
+    bundle exec rake assets:clean
+    bundle exec rake assets:precompile
   fi
-  chown -R web:web $DEPLOY_PATH
+  chown -R web:web $APP_PATH
 fi
 
-su -c "cd $DEPLOY_PATH && bundle exec puma -C config/puma.rb" -s /bin/bash -l web
+cd $APP_PATH
+bundle exec puma -C config/puma.rb
 bash

config/puma.rb

@@ -15,11 +15,11 @@ rackup DefaultRackup
 
 # Set vars as we cannmot load them
 rails_env = ENV['RAILS_ENV'] || 'development'
-app_dir = ENV['DEPLOY_PATH'] || '/var/www'
+app_dir = ENV['APP_PATH'] || '/var/www'
 
 # Set basic puma settings
 environment rails_env
-bind "unix://#{app_dir}/tmp/sockets/puma.sock"
+bind "unix://#{app_dir}/tmp/sockets/puma.#{rails_env}.sock"
 port Integer(ENV['PUMA_PORT'] || 4000)
 
 # Redirect stdout only in production. Dev mode needs it for debug
@@ -50,6 +50,6 @@ end
 # EXPLAIN This has been added here but why?
 on_restart do
   ENV["BUNDLE_GEMFILE"] = "#{app_dir}/Gemfile"
-  Dotenv.overload
+  Dotenv.overload('.env.' + ENV['RAILS_ENV'] + '.local', '.env.local', '.env.' + ENV['RAILS_ENV'], '.env')
   ActiveRecord::Base.connection.disconnect!
 end

db/initdb.d/99_ensl.setup.sql

@@ -1,4 +1,7 @@
 -- FIXME: this should be somewhere else probably
+CREATE DATABASE ensl_development;
+GRANT ALL PRIVILEGES ON ensl_development.* TO 'ensl'@'%' WITH GRANT OPTION;
+
 CREATE DATABASE ensl_test;
 GRANT ALL PRIVILEGES ON ensl_test.* TO 'ensl'@'%' WITH GRANT OPTION;
 

docker-compose.yml

@@ -2,15 +2,61 @@ version: "3.4"
 
 services:
   #
-  # Main service
+  # Main services
   #
-  web:
+  production:
+    command: "bundle exec puma"
+    tty: true
+    stdin_open: true
+    container_name: "ensl_production"
+    user: "web:web"
+    build:
+      context: ./
+      dockerfile: Dockerfile
+      target: ensl_production
+    volumes:
+     - ".:/var/www/"
+    environment:
+      RAILS_ENV: production
+    ports:
+     - "${PUMA_PORT}:${PUMA_PORT}"
+     # Guard livereload
+     # - "35729:35729"
+    links:
+     - db
+     - memcached
+     - smtp
+
+  staging:
+    # command: "/bin/bash"
+    # command: "/var/www/bin/script/entry.sh"
+    command: "bundle exec puma"
+    tty: true
+    stdin_open: true
+    container_name: "ensl_staging"
+    user: "web:web"
+    build:
+      context: ./
+      dockerfile: Dockerfile
+      target: ensl_staging
+    volumes:
+     - ".:/var/www/"
+    environment:
+      RAILS_ENV: staging
+    ports:
+     - "9000:${PUMA_PORT}"
+    links:
+     - db
+     - memcached
+     - smtp
+
+  development:
     # For debug; Use attach rather than this
     # command: /bin/bash
     command: "bundle exec puma"
     tty: true
     stdin_open: true
-    container_name: "ensl_${RAILS_ENV}"
+    container_name: "ensl_development"
     user: "web:web"
     build:
       context: ./
@@ -26,27 +72,6 @@ services:
     links:
      - db
      - memcached
-     - smtp
-     #- spring
-     #- redis
-
-  nginx:
-    image: nginx:latest
-    container_name: ensl_${RAILS_ENV}_nginx
-    command: /bin/bash -c "envsubst '$$PUMA_PORT' < /etc/nginx/conf.d/nginx.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
-    volumes:
-      - ./ext/ssl/fullchain.pem:/etc/ssl/certs/ensl_fullchain.pem
-      - ./ext/ssl/privkey.pem:/etc/ssl/private/ensl_privkey.pem
-      - ./ext/nginx.conf.d/:/etc/nginx/conf.d/
-      - ./public:/var/www/public
-    #ports:
-    #  - $APP_PORT:80
-    #  - $APP_PORT_SSL:443
-    environment:
-      - APP_DOMAIN=$APP_DOMAIN
-      - APP_PORT=$APP_PORT
-      - PUMA_PORT=$PUMA_PORT
-      - RAILS_ENV=$RAILS_ENV
 
   #
   # Testing
@@ -120,8 +145,32 @@ services:
     volumes:
      - "./ext/dkim:/etc/opendkim/keys"
     environment:
-      - POSTFIX_myhostname=$APP_DOMAIN
+      - POSTFIX_myhostname=$MAIL_DOMAIN
-      - OPENDKIM_DOMAINS=$APP_DOMAIN
+      - OPENDKIM_DOMAINS=$MAIL_DOMAIN
+
+  nginx:
+    image: nginx:latest
+    container_name: ensl_nginx
+    tty: true
+    stdin_open: true
+    command: /bin/bash -c "envsubst '$$PUMA_PORT $$APP_PATH $$APP_PATH_PUBLIC $$STAGING_ROOT_DOMAIN $$STAGING_DOMAIN $$STAGING_PORT $$STAGING_PORT_SSL' < /etc/nginx/conf.d/*.conf.template > /etc/nginx/conf.d/default.conf && cat /etc/nginx/conf.d/*.conf && nginx -g 'daemon off;'"
+    volumes:
+      - ./ext/ssl/fullchain.pem:/etc/ssl/certs/ensl_fullchain.pem
+      - ./ext/ssl/privkey.pem:/etc/ssl/private/ensl_privkey.pem
+      - ./ext/nginx.conf.d/:/etc/nginx/conf.d/
+      - ./public:/var/www/public
+    ports:
+      - $STAGING_PORT:$STAGING_PORT
+      - $STAGING_PORT_SSL:$STAGING_PORT_SSL
+    environment:
+      - PUMA_PORT=$PUMA_PORT
+      - STAGING_ROOT_DOMAIN=$STAGING_ROOT_DOMAIN
+      - STAGING_DOMAIN=$STAGING_DOMAIN
+      - STAGING_PORT=$STAGING_PORT
+      - STAGING_PORT_SSL=$STAGING_PORT_SSL
+      - RAILS_ENV=$RAILS_ENV
+      - APP_PATH=$APP_PATH
+      - APP_PATH_PUBLIC=$APP_PATH_PUBLIC
 
 #  spring:
 #    build:

ext/nginx.conf.d/staging.conf.template

@@ -3,36 +3,39 @@
 # Use it in production or copy it over
 
 upstream puma {
-  server localhost:$PUMA_PORT;
+  server staging:$PUMA_PORT;
   # server unix:/var/tmp/puma.$RAILS_ENV.sock fail_timeout=0;
 }
 
 # root-level -> www redirect
-server {
+#server {
-  listen *:80;
+#  listen *:$STAGING_PORT;
-  listen *:443 ssl;
+#  listen *:$STAGING_PORT_SSL ssl;
 
+  # FIXME
   # ssl_certificate /etc/ssl/certs/ensl_fullchain.pem;
   # ssl_certificate_key /etc/ssl/private/ensl_privkey.pem;
   # ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 
-  server_name ensl.org;
+#  #server_name $STAGING_ROOT_DOMAIN;
-  root /var/www/public;
+#  #root $APP_PATH_PUBLIC;
-  return 301 https://www.ensl.org$request_uri;
+#  return 301 https://$STAGING_DOMAIN$request_uri;
-}
+#}
 
 # HTTP -> HTTPS redirect
 #server {
-#  listen *:80;
+#  listen *:STAGING_PORT;
-#  server_name www.ensl.org;
+#  server_name $STAGING_DOMAIN;
-#  return 301 https://www.ensl.org$request_uri;
+#  return 301 https://$STAGING_DOMAIN$request_uri;
 #}
 
 server {
-  # listen *:443 ssl default_server;
+  listen *:$STAGING_PORT default_server;
-  listen *:80 default_server;
+  # SSL disabled for now
-  # server_name www.ensl.org;
+  # listen *:STAGING_PORT_SSL ssl default_server;
-  root /var/www/public;
+  # domain_agnostic staging
+  # server_name $STAGING_DOMAIN;
+  root $APP_PATH_PUBLIC;
   index index.html index.htm index.php;
 
   # ssl_certificate /etc/ssl/certs/ensl_fullchain.pem;
@@ -60,14 +63,18 @@ server {
     allow all;
     autoindex on;
   }
+
+  # FIXME: use env. var
   location ^~ /assets/ {
     gzip_static on;
     expires max;
     add_header Cache-Control public;
   }
+
+  # FIXME: use env. var
   location /files/ {
     # try_files $uri $uri/ @puma;
-    # alias root /var/www/public/files/;
+    # alias root $APP_PATH_PUBLIC/files/;
     autoindex on;
   }
   location @puma {

script/env.sh

@@ -13,6 +13,7 @@ do
   ARGS=$(cat $FILE |grep -vE '^[[:space:]]*(#.*)*$')
 
   export $(echo $ARGS|xargs)
-  echo "$ARGS\n" 
+  echo "$ARGS"
+  echo
 done