Finish recreate, Fix file browsing

Add it to adminpanel

app/assets/stylesheets/themes/default/layout/_body.scss

@@ -131,4 +131,8 @@ h1, h2, h3, h4, h5, h6 {
   ol {
     list-style-type: decimal;
   }
+
+  pre {
+    font-size: $base-font-size * 0.5;
+  }
 }

app/controllers/data_files_controller.rb

@@ -74,7 +74,7 @@ class DataFilesController < ApplicationController
   def destroy
     raise AccessError unless @file.can_destroy? cuser
     @file.destroy
-    redirect_to_back
+    redirect_to directory_path(@file.directory)
   end
 
   def rate

app/controllers/directories_controller.rb

@@ -21,13 +21,9 @@ class DirectoriesController < ApplicationController
   end
 
   def recreate
-    @directory.recreate_transaction
-    render text: t(:directories_update)
-  end
-
-  def refresh
-    @directory.process_dir
-    render text: t(:directories_update)
+    raise AccessError unless @cuser&.admin?
+    @result = @directory.recreate_transaction
+    @nobody = true
   end
 
   def create
@@ -55,7 +51,7 @@ class DirectoriesController < ApplicationController
   def destroy
     raise AccessError unless @directory.can_destroy? cuser
     @directory.destroy
-    redirect_to directories_url
+    redirect_to directory_path(Directory.find(Directory::ROOT))
   end
 
   private

app/helpers/application_helper.rb

@@ -22,7 +22,7 @@ module ApplicationHelper
     stylesheet_link_tag "themes/#{active_theme}/theme"
   end
 
-  def namelink model, length = nil
+  def namelink(model, length = nil)
     return if model.nil?
     model = case model.class.to_s
             when "DataFile"
@@ -38,12 +38,23 @@ module ApplicationHelper
 
     # Reduce length of too long model names
     if length and str.length > length
-      link_to str.to_s[0, length] + "...", model, class: model.class.to_s.downcase
+      link_to(str.to_s[0, length] + "...", model, class: model.class.to_s.downcase)
     else
-      link_to str, model, class: model.class.to_s.downcase
+      link_to(str, model, class: model.class.to_s.downcase)
     end
   end
 
+  def directory_links(directory)
+    output = ""
+    Directory.directory_traverse(directory).reverse_each do |dir|
+      output << namelink(dir) + "\n"
+      unless dir == directory
+        output << " &raquo; \n"
+      end
+    end
+    output.html_safe
+  end
+
   def shorten str, length
     if length and str and str.to_s.length > length
       str = str.to_s[0, length] + "..."

app/models/data_file.rb

@@ -83,6 +83,12 @@ class DataFile < ActiveRecord::Base
     name.url
   end
 
+  def manual_upload(manual_location)
+    File.open(manual_location) do |f|
+      self.name = f
+    end
+  end
+
   def process_file
     self.md5 = "e948c22100d29623a1df48e1760494df"
 
@@ -90,7 +96,7 @@ class DataFile < ActiveRecord::Base
       self.directory_id = Directory::ARTICLES
     end
 
-    if File.exists?(location) and (size != File.size(location) or created_at != File.mtime(location))
+    if File.exists?(location) and (new_record? or size != File.size(location) or created_at != File.mtime(location))
       self.md5 = Digest::MD5.hexdigest(File.read(location))
       self.size = File.size(location)
       self.created_at = File.mtime(location)
@@ -147,9 +153,11 @@ class DataFile < ActiveRecord::Base
     user and !rated_by?(user)
   end
 
-  def self.find_existing(subdir_name, subitem_path)
-    DataFile.where(arel_tabe(:path).eq(subitem_path)\
-      .or(arel_table(:md5).eq(Digest::MD5.hexdigest(File.read(subitem_path))))).first
+  # TODO: instead of using path, use name + directory path
+  def self.find_existing(subitem_path, subitem_name)
+    hash = Digest::MD5.hexdigest(File.read(subitem_path))
+    DataFile.where(arel_table[:path].eq(subitem_path)\
+      .or(arel_table[:md5].eq(hash))).first
   end
 
   def can_create? cuser

app/models/directory.rb

@@ -17,19 +17,16 @@
 #
 #  index_directories_on_parent_id  (parent_id)
 #
+require 'stringio'
 
 ENV['FILES_ROOT'] ||= File.join(Rails.root, 'public', 'files')
 
-# class SteamIdValidator < ActiveModel::Validator
-#   def validate(record)
-#     record.errors.add :steamid unless \
-#       record.steamid.nil? ||
-#         (m = record.steamid.match(/\A([01]):([01]):(\d{1,10})\Z/)) &&
-#         (id = m[3].to_i) &&
-#         id >= 1 && id <= 2147483647
-#   end
-# end
-
+class PathValidator < ActiveModel::Validator
+  def validate(record)
+    record.errors.add :path, "doesn't match generated path" unless \
+      record.full_path == record.path
+  end
+end
 
 class Directory < ActiveRecord::Base
   include Extra
@@ -41,43 +38,84 @@ class Directory < ActiveRecord::Base
   MOVIES = 30
   ARTICLES = 39
 
-  #attr_protected :id, :updated_at, :created_at, :path
+  attr_accessor :preserve_files
 
   belongs_to :parent, :class_name => "Directory", :optional => true
   has_many :subdirs, :class_name => "Directory", :foreign_key => :parent_id
   has_many :files, -> { order("name") }, :class_name => "DataFile"
 
   scope :ordered, ->  { order("name ASC") }
+  scope :path_sorted, ->  { order("path ASC") }
   scope :filtered, -> { where(hidden: false) }
   scope :of_parent, -> (parent) { where(parent_id: parent.id) }
 
-  validates_length_of [:name, :path], :in => 1..255
+  # FIXME: different validation for user?
+  validates_length_of [:name, :path, :title], :in => 1..255
   validates_format_of :name, :with => /\A[A-Za-z0-9]{1,20}\z/, :on => :create
-  validates_length_of :name, :in => 1..25
+  validates_length_of :name, :in => 1..255
   validates_inclusion_of :hidden, :in => [true, false]
+  validates_presence_of :title
+  validates_with PathValidator
   # TODO: add validation for path
 
-  before_validation :init_variables
+  before_validation :init_variables, on: :create
   after_create :make_path
   after_save :update_timestamp
-  before_destroy :remove_files
+  before_destroy :remove_files, unless: Proc.new { preserve_files }
   after_destroy :remove_path
 
   def to_s
     name
   end
 
-  def init_variables
-    self.path = full_path if parent
-    self.hidden = false if hidden.nil?
+  def parent_root?
+    parent.id == Directory::ROOT
+  end
+
+  def root?
+    id == Directory::ROOT
+  end
+
+  def full_title
+    output = ""
+    Directory.directory_traverse(self).reverse_each do |dir|
+      unless dir.title&.empty?
+        output << "%s" % dir.title
+      else
+        output << dir.name
+      end
+      output << " » " unless self == dir
+    end
+    output
+  end
+
+  def self.directory_traverse(directory, list = [])
+    unless directory.root?
+      list << directory
+      return directory_traverse(directory.parent, list)
+    else
+      return list
+    end
+  end
+
+  # Use this
+  def full_path
+    parent ? File.join(parent.full_path, name.downcase) : path
+  end
+
+  def relative_path
+    parent ? File.join(parent.relative_path, name.downcase).sub(/^\//, '') : ""
   end
 
   def path_exists?
     File.directory?(full_path)
   end
 
-  def full_path
-    parent ? File.join(parent.full_path, name.downcase) : path
+  def init_variables
+    # Force path to use parent which is the authoritative source
+    self.path = full_path if parent
+    self.title = File.basename(self.path).capitalize
+    self.hidden = false if hidden.nil?
   end
 
   def make_path
@@ -93,6 +131,7 @@ class Directory < ActiveRecord::Base
       subdir.destroy
     end
     subdirs.each do |subdir|
+      subdir.preserve_files = self.preserve_files
       subdir.destroy
     end
   end
@@ -103,65 +142,86 @@ class Directory < ActiveRecord::Base
 
   # TODO: make tests for this, moving etc.
   # TODO: mutate instead of return.
-  def recreate_transaction(root = ENV['FILES_ROOT'])
-    logger = Rails.logger
-    logger.info 'Starting recreate on %d, root: %s' % [id, root]
+  # TODO: move to its own class
+  # TODO: also remove files
+  # TODO: need log to rails log too
+  def recreate_transaction
+    strio = StringIO.new
+    logger = Logger.new(strio)
+    logger.info 'Starting recreate on Directory(%d): %s.' % [id, name]
     ActiveRecord::Base.transaction do
       # We use destroy lists so technically there can be seperate roots
       destroy_dirs = Hash.new
-      update_attribute :path, root
-      destroy_dirs = recreate(destroy_dirs)
+      if id == Directory::ROOT
+        update_attribute :path, ENV['FILES_ROOT']
+      end
+      logger.info 'Path: %s' % [path]
+      destroy_dirs = recreate(destroy_dirs, logger: logger)
       destroy_dirs.each do |key, dir|
         logger.info 'Removed dir: %s' % dir.full_path
-        # dir.destroy!
+        dir.destroy!
       end
     end
     logger.info 'Finish recreate'
-    return nil
+    return strio
     # TODO: check items that weren't checked.
   end
 
   # QUESTION Symlinks?
-  def recreate(destroy_dirs, path = self.full_path)
+  def recreate(destroy_dirs, logger: Rails.logger)
     # Convert all subdirs into a hash and mark them to be deleted
     # FIXME: better oneliner
+    # logger.debug 'recreate: %s' % full_path
     destroy_dirs.merge!(subdirs.all.map{ |s| [s.id,s] }.to_h)
 
     # Go through all subdirectories (no recursion)
-    Dir.glob("%s/*" % path).each do |subitem_path|
-      if File.directory? subitem_path
-        subdir_name = File.basename(subitem_path)
+    Dir.glob(File.join(full_path, '*')).each do |subitem_path|
+      subitem_name = File.basename(subitem_path)
 
+      if File.directory? subitem_path
+        # logger.debug 'Processing dir: %s' % subitem_path
         # We find by name only, ignore path
         # Find existing subdirs from current path. Keep those we find
-        if (subdir = find_existing(subdir_name, subitem_path))
+        if (subdir = find_existing(subitem_name, subitem_path))
           if subdir.parent_id != self.id
             old_path = subdir.full_path
             subdir.parent = self
             subdir.save!
             logger.info 'Renamed dir: %s -> %s' % [old_path, subdir.full_path]
+          elsif !subdir.valid?
+            subdir.errors.full_messages.each do |err|
+              logger.error err
+            end
+            subdir.init_variables
+            logger.info 'Fixed attributes: %s' % [subdir.full_path]
+            subdir.save!
           end
           destroy_dirs.delete subdir.id
         # In case its a new directory
         else
           # Attempt to find it in existing directories
-          subdir = subdirs.build(name: subdir_name)
+          subdir = subdirs.build(name: subitem_name)
           # FIXME: find a better solution
           subdir.save!(validate: false)
           logger.info 'New dir: %s' % subdir.full_path
         end
-
         # Recreate the directory
-        destroy_dirs = subdir.recreate(destroy_dirs)
+        destroy_dirs = subdir.recreate(destroy_dirs, logger: logger)
       elsif File.file? subitem_path
+        # logger.debug 'Processing file: %s' % subitem_path
         if dbfile = DataFile.find_existing(subitem_path, subitem_name)
+          if dbfile.directory_id != self.id
             dbfile.directory = self
             dbfile.save!
-        elsif (File.mtime(file) + 100).past?
+            logger.info 'Update file: %s' % dbfile.name
+          end
+        elsif (File.mtime(subitem_path) + 100).past?
           dbfile = DataFile.new
-          dbfile.path = file
+          # dbfile.name = subitem_name
           dbfile.directory = self
+          dbfile.manual_upload(subitem_path)
           dbfile.save!
+          logger.info 'Added file: %s' % dbfile.name
         end
         # TODO: handle files that are only in database
       end
@@ -181,7 +241,7 @@ class Directory < ActiveRecord::Base
         end
       end
       # TODO: use filter_map here
-      # NOTE: we don't use the logic from dat_file
+      # NOTE: we don't use the logic from date_file
       file_count = Dir["%s/*" % subitem_path].count{|f| File.file?(f) }
       Directory.joins(:files).group('data_files.directory_id')\
         .having('count(data_files.id) = ? and count(data_files.id) > 0', file_count).each do |dir|
@@ -195,11 +255,6 @@ class Directory < ActiveRecord::Base
     return false
   end
 
-  # TODO
-  def recreate_check
-    true
-  end
-
   # TODO check that you can download files
   
   def can_create? cuser

app/uploaders/file_uploader.rb

@@ -4,7 +4,8 @@ class FileUploader < CarrierWave::Uploader::Base
   # Override the directory where uploaded files will be stored.
   # This is a sensible default for uploaders that are meant to be mounted:
   def store_dir
-    model.directory.path.gsub(/public\//, '')
+    model.directory.full_path
+    # .gsub(/public\//, '')
   end
 
   # Provide a default URL as a default if there hasn't been a file uploaded:

app/views/about/adminpanel.html.erb

@@ -2,25 +2,53 @@
     <h1>
         Admin Menu
     </h1>
+    <h4>
+      Articles
+    </h4>
     <ul class="disc">
         <li><%= link_to 'New Article', new_article_path %></li>
         <li><%= link_to "Article Admin", controller: "articles", action: "admin" %></li>
+    </ul>
+
+    <h4>
+      Files
+    </h4>
+    <ul class="disc">
         <% if Directory.first %>
+        <li><%= link_to "Recreate Root", recreate_directory_path(Directory.find(Directory::ROOT)) %></li>
         <li><%= link_to "Files Admin", controller: "directories", action: "show", id: Directory.first %></li>
         <% end %>
+    </ul>
+
+    <h4>
+      Contests
+    </h4>
+    <ul class="disc">
+        <li><%= link_to "Contests", contests_path %></li>
+        <li><%= link_to "Challenges", challenges_path %></li>
+        <li><%= link_to "Maps", maps_path %></li>
+    </ul>
+
+    <h4>
+      Users and groups
+    </h4>
+    <ul class="disc">
+        <li><%= link_to "Users", users_path %></li>
+        <li><%= link_to "Groups", groups_path %></li>
+        <li><%= link_to "Bans", bans_path %></li>
+    </ul>
+
+    <h4>
+      Website
+    </h4>
+    <ul class="disc">
+        <li><%= link_to "Categories", categories_path %></li>
+        <li><%= link_to "Custom Article URLs", custom_urls_path %></li>
         <li>
             <%= link_to issues_path do %>
             Issues (<%= Issue.with_status(0).count %>)
             <% end %>
         </li>
-        <li><%= link_to "Bans", bans_path %></li>
-        <li><%= link_to "Groups", groups_path %></li>
-        <li><%= link_to "Categories", categories_path %></li>
         <li><%= link_to "Polls", polls_path %></li>
-        <li><%= link_to "Contests", contests_path %></li>
-        <li><%= link_to "Challenges", challenges_path %></li>
-        <li><%= link_to "Maps", maps_path %></li>
-        <li><%= link_to "Users", users_path %></li>
-        <li><%= link_to "Custom Article URLs", custom_urls_path %></li>
     </ul>
 </div>

app/views/data_files/edit.html.erb

@@ -12,7 +12,7 @@
 
   <div class="fields horizontal">
     <%= f.label :directory_id %>
-    <%= f.select :directory_id, Directory.all.collect { |c| ["#{c.path} - #{c.name}", c.id] } %>
+    <%= f.select :directory_id, Directory.path_sorted.collect { |c| ["#{c.full_title} (#{c.relative_path})", c.id] } %>
   </div>
 
   <div class="controls">

app/views/directories/_form.html.erb

@@ -12,7 +12,7 @@
     <% end %>
 
     <p>
-      <b>Parent:</b> <%= @directory.parent.path %>
+      <b>Parent:</b> <%= @directory.parent.full_path %>
   </p>
 
   <p>

app/views/directories/recreate.html.erb

@@ -0,0 +1,5 @@
+<%= t(:directories_update) %>
+
+<pre>
+  <%= @result.string %>
+</pre>

app/views/directories/show.html.erb

@@ -1,5 +1,13 @@
 <h1 class="title">Files</h1>
 
+<p>
+<% if @directory.root? %>
+  In case there are any problems with this browser, you can find files
+  <%= link_to "here", "/files/" %>
+  too.
+<% end%>
+</p>
+
 <%
   active = 1
   n = 1
@@ -15,9 +23,8 @@
   <div class="tabbed-contents">
     <% @directories.each do |dir| %>
       <div class="tab" id="dir_<%= dir.id %>">
-      
         <%
-          if @directory.path.include?(dir.path)
+          if @directory.full_path.include?(dir.full_path)
             dir = @directory
             active = n
           end
@@ -25,22 +32,21 @@
         %>
 
         <div class="directories">
-          <% if !dir.subdirs.ordered.empty? or dir.parent.id != Directory::ROOT %>
-            <h3>Directories</h3>
+        <% if !dir.parent_root? %>
+          <h3><%= directory_links @directory %></h3>
+        <% end %>
+        <% if !dir.subdirs.ordered.empty?  %>
+          <% if dir.parent_root?  %>
+            <h3>Sub-directories</h3>
           <% end %>
-
             <div class="subdirectories">
               <ul class="disc">
-              <% if dir.parent.id != Directory::ROOT %>
-                <li class="parent">
-                  <%= link_to "Parent", dir.parent %>
-                </li>
-              <% end %>
                 <% dir.subdirs.ordered.each do |subdir| %>
-                <li><%= namelink subdir %></li>
+                  <li><%= link_to subdir.name, subdir %></li>
                 <% end %>
               </ul>
             </div>
+          <% end %>
         </div>
 
         <h3>Files</h3>
@@ -77,7 +83,7 @@
         <% if cuser and cuser.admin? %>
           <div class="controls">
             <%= link_to "Edit Directory", edit_directory_path(dir), class: 'button' %>
-            <%= link_to "Delete Directory", dir, confirm: "Are you REALLY sure?", method: :delete, class: 'button' %>
+            <%= link_to "Delete Directory", dir,  data: { confirm: "Are you sure?" }, class: 'button' %>
             <%= link_to "New Directory", { controller: "directories", action: "new", id: dir }, { class: 'button' } %>
             <%= link_to "New File", { controller: "data_files", action: "new", id: dir }, { class: 'button' } %>
           </div>

config/routes.rb

@@ -85,7 +85,11 @@ Ensl::Application.routes.draw do
   resources :maps
   resources :logs
   resources :log_files
-  resources :directories
+  resources :directories do
+    member do
+      get :recreate
+    end
+  end
   resources :data_files
   resources :predictions
   resources :weeks