path: root/lib/gorg/cache.rb

###   Copyright 2004,   Xavier Neys   (neysx@gentoo.org)
# #
# #   This file is part of gorg.
# #
# #   gorg is free software; you can redistribute it and/or modify
# #   it under the terms of the GNU General Public License as published by
# #   the Free Software Foundation; either version 2 of the License, or
# #   (at your option) any later version.
# #
# #   gorg is distributed in the hope that it will be useful,
# #   but WITHOUT ANY WARRANTY; without even the implied warranty of
# #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# #   GNU General Public License for more details.
# #
# #   You should have received a copy of the GNU General Public License
# #   along with Foobar; if not, write to the Free Software
###   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


# Cache a bit of data based on 
#  . a path name as received by a webserver e.g.
#  . a list of parameters as received by a webserver e.g.
#  . a list of files it depends on

begin
  require "parsedate"
  $haveparsedate = true
rescue LoadError
  require "time"
  $haveparsedate = false
end
require "fileutils"
require "find"
require "digest"
require "digest/md5"

module Gorg

CacheStamp = "Gorg-#{Gorg::Version} Cached This Data. Do not alter this file. Thanks."

module Cache
  def Cache.init(config)
    @@lockfile = ".cache.cleaner.lock"
    @cacheDir = nil
    if FileTest.directory?(config["cacheDir"])
      if FileTest.writable?(config["cacheDir"])
        @cacheDir = config["cacheDir"].chomp("/")
      else
        warn "Cache directory not writable"
      end
    else
      warn "Invalid cache directory"
    end

    # Time-To-Live in seconds, cached items older than that will be considered too old
    @zipLevel = config["zipLevel"]
    @zip = @zipLevel > 0 ? ".gz" : ""
    @ttl = config["cacheTTL"]
    @cacheTree = config["cacheTree"]
    @maxFiles = config["maxFiles"]            # Max number of files in a single directory
    @maxSize = config["cacheSize"]*1024*1024  # Now in bytes
    @washNumber = config["cacheWash"]         # Clean cache dir after a store operation whenever rand(@washNumber) < 10
    @lastCleanup = Time.new-8e8               # Remember last time we started a cleanup so we don't pile them up
  end
  
  def Cache.hit(objPath, objParam={}, etags=nil, ifmodsince=nil)
    # objPath is typically a requested path passed from a web request but it
    # can be just any string. It is not checked against any actual files on the file system
    #
    # objParam is expected to be a hash or any object whose iterator yields two values
    #
    # 2 filenames are built with the arguments and should give 
    # the name of a metafile and a result file
    # if the result file is older than @ttl seconds, hit fails
    # The metafile is then checked for dependencies
    # It contains a list of filenames along with their size and mtime separated by ;;
    
    # etag and ifmodsince are used in a webserver context
    #   etag is defined if an ETag was part of an If-None-Match request field
    #   etag can be an array or a single string
    #   If the current ETag of the meta file matches, no data is returned (webserver should return a 304)
    #
    #   ifmodsince is a time object passed on an If-Modified-Since request field
    #   If the creation date of the meta file is earlier, no data is returned (webserver should return a 304)

    return nil if @cacheDir.nil? # Not initialized, ignore request
    
    # Reminder: filenames are full path, no need to prepend dirname
    dirname, basename, filename, metaname = makeNames(objPath, objParam)
    
    raise "Cache subdir does not exist" unless FileTest.directory?(dirname)

    # Hit the cache
    meta, mstat = IO.read(metaname), File.stat(metaname)  if metaname && FileTest.file?(metaname) && FileTest.readable?(metaname)
    raise "Empty/No meta file" if meta.nil? || meta.length < 1

    fstat = File.stat(filename) if filename && FileTest.file?(filename)
    raise "Empty/No data file" if fstat.nil?

    # Check the timestamps of files in the metadata
    meta = meta.split("\n")
    raise "I did not write that meta file" unless CacheStamp == meta.shift
    mline = meta.shift
    while mline and mline !~ /^;;extra meta$/ do
      f, s, d = mline.split(";;")
      if s.to_i < 0
        # File did not exist when cache entry was created
        raise "Required file #{f} has (re)appeared" if FileTest.file?(f) && FileTest.readable?(f)
      else
        # File did exist when cache entry was created, is it still there?
        raise "Required file #{f} has disappeared" unless FileTest.file?(f) && FileTest.readable?(f)
      
        fst = File.stat(f)
        raise "Size of #{f} has changed from #{fst.size} to #{s.to_i}" unless fst.size == s.to_i
        if $haveparsedate
          raise "Timestamp of #{f} has changed" unless Time.utc(*ParseDate.parsedate(d)) == fst.mtime.utc
        else
          raise "Timestamp of #{f} has changed" unless Time.parse(d) == fst.mtime.utc
        end
      end
      mline = meta.shift
    end
    if mline =~ /^;;extra meta$/ then
      extrameta = meta.dup
    else
      extrameta = []
    end
    
    if notModified?(fstat, etags, ifmodsince) and extrameta.join !~ /set-cookie/i
      raise Gorg::Status::NotModified.new(fstat)
    end
    
    file = IO.read(filename) if filename && FileTest.file?(filename) && FileTest.readable?(filename)
    raise "Empty/No data file" if file.nil? || file.length < 1

    # Is the data file too old
    raise "Data file too old" unless @ttl==0 or (Time.new - fstat.mtime) < @ttl
    
    # Update atime of files, ignore failures as files might have just been removed
    begin
      t = Time.new
      File.utime(t, fstat.mtime, filename)
      File.utime(t, mstat.mtime, metaname)
    rescue
      nil
    end
    
    # If we get here, it means the data file can be used, return cache object (data, stat(datafile), extrameta)
    # The file is left (un)compressed, it's returned as it was stored
    [file, fstat, extrameta]
    
  rescue Gorg::Status::NotModified
    # Nothing changed, should return a 304
    debug("Client cache is up-to-date")
    raise
  rescue
    # cache hit fails if anything goes wrong, no exception raised
    debug("Cache hit on #{objPath} failed: (#{$!})")
    nil
  end


  def Cache.store(data, objPath, objParam={}, deps=[], extrameta=[])
    # Store data in cache so it can be retrieved based on the objPath and objParams
    # deps should contain a list of files that the object depends on
    # as returnd by our xsl processor, i.e. an array of [access_type, path] where
    # access_type can be "r", "w", or "o" for recpectively read, write, other.

    # Define content-type
    ct = setContentType(data)
    extrameta << "Content-Type:#{ct}"
    
    return nil if @cacheDir.nil? # Not initialized, ignore request
    
    # Cache only if no remote objects (ftp:// or http://) in list of used files
    if deps && deps.detect{|f| f[0] =~ /^o$/i }
      debug "#{objPath} not cached because it needs remote resources"
      return nil
    end

    dirname, basename, filename, metaname = makeNames(objPath, objParam)

    FileUtils.mkdir_p(dirname) unless FileTest.directory?(dirname)
    
    # Write Meta file to a temp file (with .timestamp.randomNumber appended)
    metaname_t = "#{metaname}.#{Time.new.strftime('%Y%m%d%H%M%S')}.#{rand(9999)}"

    # Data might need to be just a link to another .Data file
    # if we find another requested path with different params but
    # with identical MD5 sums
    # Which is why we keep a ...xml.Data.[md5 sum] file without the parameters
    # in its name that we can hard link to.
    # e.g. A moron hits for 10 full handbooks with toto=1..10 in the URI,
    # we'd end up with 10 identical large copies. With links we have only one

    # Old versions are expected to be cleaned up by the cacheWash() routine
    # A Dir.glob() to find the previous ones would be too expensive
    
    # Compute MD5 digest
    md5 = Digest::MD5.hexdigest(data)
    
    # Compress data if required
    if @zipLevel > 0 then
      bodyZ = data = gzip(data, @zipLevel)
    else
      bodyZ = nil
    end
    
    # Set mtime of data file to latest mtime of all required files
    # so that caching can work better because mtimes will be
    # identical on all webnodes whereas creation date of data
    # would be different on all nodes.
    maxmtime = Time.now-8e8
    fstat = nil
    
    begin
      timeout(10){
        File.open("#{metaname_t}", "w") {|fmeta|
          fmeta.puts(CacheStamp)
          # Write filename;;size;;mtime for each file in deps[]
          deps.each {|ffe|
            ftype = ffe[0]
            fdep = ffe[1]
            if FileTest.file?(fdep)
              s = File.stat(fdep)
              fmeta.puts("#{fdep};;#{s.size};;#{s.mtime.utc};;#{ftype}")
              maxmtime = s.mtime if s.mtime > maxmtime and ftype =~ /^r$/i
            else
              # A required file does not exist, use size=-1 and old timestamp
              # so that when the file comes back, the cache notices a difference
              # and no cache miss gets triggered as long as file does not exist
              fmeta.puts("#{fdep};;-1;;Thu Nov 11 11:11:11 UTC 1971")
            end
          }
          fmeta.puts ";;extra meta"
          extrameta.each { |m| fmeta.puts m }
        }
        # Get exclusive access to the cache directory while moving files and/or creating data files
        File.open(dirname) { |lockd|
          while not lockd.flock(File::LOCK_NB|File::LOCK_EX)
            # Timeout does not occur on a blocking lock
            # Try a non-bloking one repeatedly for a few seconds until timeout occurs or lock is granted
            # We are in a timeout block, remember
            sleep 0.1
          end
          # Remove previous Data
          FileUtils.rm_rf(filename)

          # mv temp meta file to meta file
          FileUtils.mv(metaname_t, metaname)

          # We keep a data file for the same requested path, with different params,
          # but which ends up with same MD5 sum, i.e. identical results because of unused params
          linkname = "#{basename}.#{md5}#{@zip}"
          if FileTest.file?(linkname) then
            # Data file already there, link to it
            File.link(linkname, filename)
          else
            # Write data file and set its mtime to latest of all files it depends on
            File.open("#{filename}", "w") {|fdata| fdata.write(data)}
            # Create link
            File.link(filename, linkname)
          end
          # mtime might need to be updated, or needs to be set
          # e.g. when a dependency had changed but result files is identical
          # This is needed to keep Last-Modified dates consistent across web nodes
          File.utime(Time.now, maxmtime, filename)
          fstat = File.stat(filename)
        }
      }
    ensure
      FileUtils.rm_rf(metaname_t)
    end
    
    # Do we clean the cache?
    washCache(dirname, 10) if @washNumber > 0 and rand(@washNumber) < 10
    
    # Return stat(datafile) even if it's just been removed by washCache
    # because another web node might still have it or will have it.
    # Anyway, the cached item would be regenerated on a later request
    # and a 304 would be returned if still appropriate at the time.

    # Return fstat of data file (for etag...) and zipped file
    [fstat, bodyZ]
    
  rescue Timeout::Error, StandardError =>ex
    if ex.class.to_s =~ /timeout::error/i then
      warn("Timeout in cache store operation")
    else
      warn("Cache store error (#{$!})")
    end
    # Clean up before leaving
    FileUtils.rm_rf(filename||"")
    FileUtils.rm_rf(metaname||"")
    nil # return nil so that caller can act if a failed store really is a problem
  end
    
    
  def Cache.washCache(dirname, tmout=30, cleanTree=false)
    # Clean cache entries that are either too old compared to TTL (in seconds)
    # or reduce total size to maxSize (in MB)
    # oldDataOnly means to look only for unused *.Data.[md5] files that are not used anymore 
    # because file has been modified and has generated a new *.Data.[md5] file
    
    # timeout is the maximum time (in seconds) spent in here

    return nil if @cacheDir.nil? # Not initialized, ignore request
    
    # Also ignore request if dirname not equal to @cacheDir or under it
    return nil unless dirname[0, @cacheDir.length] == @cacheDir
    
    # Also ignore request if dirname does not exist yet
    return nil unless FileTest.directory?(dirname)
    
    # Also return if less than a minute has elapsed since latest cleanup
    t0 = Time.new
    return nil if t0 - @lastCleanup < 60
    
    # Remember for next time
    @lastCleanup = t0

    Dir.chdir(dirname) { |d|
      # Recreate lock file if it's been lost
      unless File.exist?(@@lockfile)
        File.open(@@lockfile, "w") { |lockf| lockf.puts("Lock file created on #{Time.now.utc} by gorg")}
      end
        
      # Grab lockfile
      File.open(@@lockfile) { |lockf| 
        if lockf.flock(File::LOCK_NB|File::LOCK_EX) then
          infoMsg = "Cleaning up cache in #{dirname} (cleanTree=#{cleanTree}, tmout=#{tmout})"
          info(infoMsg)
          puts infoMsg if cleanTree

          timeout(tmout) {
            totalSize, deletedFiles, scannedDirectories = washDir(dirname, cleanTree)
            if totalSize >= 0 then
              # Size == -1 means dir was locked, throwing an exception would have been nice :)
              infoMsg = if cleanTree then
                          "Cache in #{dirname} is now #{totalSize/1024/1024} MB, #{deletedFiles} files removed in #{(Time.now-t0).to_i} seconds in #{scannedDirectories} directories"
                        else
                          "#{deletedFiles} files removed in #{(Time.now-t0).to_i} seconds in #{dirname}"
                        end
              info(infoMsg)
              puts infoMsg if cleanTree
            end
          }
        else
          # Locked dir, another process is busy cleaning up/
          debug("#{dirname} locked, skipping")
          puts("#{dirname} locked, skipping") if cleanTree
        end # of lock test
      } # end of File.open(@@lockfile),  close & release lock automatically
    }
  rescue Timeout::Error
    info("Timeout while cleaning #{dirname}")
    puts("Timeout while cleaning #{dirname}") if cleanTree
  rescue StandardError =>ex
    error("Error while cleaning cache: #{ex}")
    puts("Error while cleaning cache: #{ex}") if cleanTree
  end

  
  private

  def Cache.washDir(dirname, cleanTree)
    # Clean up cache starting from dirname and in subdirectories if cleanTree is true
    # Return [newSize in bytes, # deleted files, # scanned directories]
    size = nDeleted = nDirectories = 0

    Dir.chdir(dirname) { |d|
      hIno = Hash.new(0) # hash of file inodes with more than one link
      lst = Array.new    # array of file names, atime, ...
      ttl = @ttl
      ttl = 8e8 if ttl == 0 # No ttl, keep very old docs!

      # Get list of files sorted on their dirname+atime
      Find.find('.') { |f|
        begin
          unless f =~ /^\.$|#{@@lockfile}/  # ignore "." and lockfile 
            ff = File.stat(f)
            if ff.directory? then
              Find.prune unless cleanTree
            elsif ff.file? and f =~ /Meta|Data/ then
              hIno[ff.ino] = ff.nlink if ff.nlink > 1
              # List of files has [name, atime, size, # links, inode]
              lst << [f, ff.atime, ff.size, ff.nlink, ff.ino]
            end
          end
        rescue
          nil # File.stat can fail because file could have been deleted, ignore error
        end
      }
      
      # Compute total size
      size = lst.inject(0){ |tot, a| tot + if a[3] > 0 then a[2]/a[3] else 0 end }
      
      # Delete old *.Data.[md5] files that are not being referenced anymore/
      lst.each { |a|
        if a[3] == 1 && a[0] =~ /\.Data\.[0-9a-f]+(.gz)?$/ then
          # Data file with no more links pointing to it
          FileUtils.rm_rf(a[0])
          nDeleted += 1
          size -= a[2]
          a[3] = 0 # Mark as deleted
        end
      }
      
      # Sort all files on atime
      lst.sort!{ |a1, a2| a1[1] <=> a2[1] }
      
      t0 = Time.new
      # Clean until size < maxSize _AND_ atime more recent than TTL
      lst.each { |a|
        break if size < @maxSize and t0-a[1] < ttl
        next if a[3] < 1 # Already deleted in previous step
        FileUtils.rm_rf(a[0])
        nDeleted += 1
        # Total size -= file size IF last link to data
        if a[3] == 1 || hIno[a[4]] <= 1 then
          size -= a[2]
        end
        hIno[a[4]] -= 1 if hIno[a[4]] > 0
        a[3] = 0 # Mark as deleted by setting nlinks to 0
      }
      
      # Remove deleted files from array
      lst.reject! { |a| a[3] < 1 }
      
      
      # Sort files per directory to enforce maxFiles
      if cleanTree then
        # Split the array in an array per directory
        # and keep the files sorted on atime in each directory
        slst = Hash.new
        lst.length.times {
          a = lst.shift
          d = File.dirname(a[0])
          if slst[d] then
            slst[d] << a
          else
            slst[d] = [a]
          end
        }
      else
        # If not cleaning whole tree, we have only a single dir
        slst = {"." => lst}
      end
      
      nDirectories = slst.length

      slst.each { |d, lst|
        # Remove oldest files so that we have less than @maxFiles in it
        if lst.length >= @maxFiles then
          # Remove to leave up 90% of #maxFiles so we don't clean up only a handful of files repeatedly
          (lst.length - 9*@maxFiles/10).times {
            if a = lst.shift then
              FileUtils.rm_rf(a[0])
              nDeleted += 1
              # Total size -= file size IF last link to data
              if a[3] == 1 || hIno[a[4]] <= 1 then
                size -= a[2]
              end
              hIno[a[4]] -= 1 if hIno[a[4]] > 0
            end
          }
        end
      }
    } #end of chdir
    [size, nDeleted, nDirectories]
  end
                    
  
  def Cache.makeNames(obj, params)
    # Build meta filename and data filename from arguments
    #
    # obj is broken into a path and a filename with appended params
    # e.g. /proj/en/index.xml?style=printable becomes /proj/en and index.xml+printable+yes
    #  or  .#proj#en#index.xml+printable+yes
    # depending on cacheTree param value

    # .Meta and .Data are appended respectively to the meta filename and data filename
    # Base is the filename without appending params, e.g. .#proj#en#index.xml.Data
    if @cacheTree then
      # Use a path and a file
      dir = "#{@cacheDir}#{File.dirname(obj)}"
      base = f = File.basename(obj)
    else
      # Convert full path into a single filename
      dir = @cacheDir
      base = f = ".#{obj.gsub(/\//,'#')}"
    end

    f = "#{f}+#{params.reject{|k,v| k.nil?}.sort.join('+')}" if params && params.to_a.length > 0    
    # Remove funky chars and squeeze duplicates into single chars
    f = f.gsub(/[^\w\#.+_-]/, "~").squeeze("~.#+")
    
    # Return names for Data and Meta files, and just the filepath (e.g. #proj#en#index.xml)
    [dir, "#{dir}/#{base}.Data", "#{dir}/#{f}.Data#{@zip}", "#{dir}/#{f}.Meta"]
  end
end

end