scan_TrainDodger_archives/scan_TrainDodger_archives.rb

#!/usr/bin/ruby

# A script to efficiently search through a JSON file of archives
# of a certain lolcow's exploits over the last two decades.
#
# Assuming the JSON file is saved as archives.json, run this script
# at the command line using
#   ./scan_TrainDodger_archives.rb archives.json
#
# The script downloads local HTML strings for all of the archives
# listed in the JSON file, and defines various functions for
# efficiently searching through the archives for interesting tid-bits.
# The script finally instantiates an IRB repl where the user has
# fine-grained control over further searching and filtering.
#
# Most useful is probably the KEYWORDS_SEARCH(archive, keywords)
# function, which peruses the archive HTML for a given set of keywords
# and returns a hash listing the archive urls and the context for
# easy reading.


require "irb"
require "json"
require "nokogiri"
require "open-uri"


# Download and return the HTML for the webpage at the given URL.
#
# Usage:
#   html = get_html_document_object("https://archive.li/XXXXX")
#     #=> Nokogiri::HTML::Document ]>
def download_html_document_object(url)
  print "Downloading from " + url + "... "
  STDOUT.flush
  begin
    html = Nokogiri::HTML.parse(URI.open(url))
  rescue OpenURI::HTTPError
    # Didn't download properly (e.g. timeout, bad read, etc)
    puts "FAIL"
    return nil
  end
  puts "OK"
  return html
end


# Return an array of archive URL strings extracted from the
# given JSON_FILE.
#
# Usage:
#   archives = extract_archives_from_json(json_file)
#   archives  #=> ["https://archive.li/XXXXX", ... ]
def extract_archives_from_json(json_file)
  json_string = IO.read(json_file)
  JSON::parse(json_string).map { |archive| archive["archived"] }
end


# Return an array of all indices at which a given WORD appears in
# the given TEXT string. The search is case-insensitive.
#
# Usage:
#   find_word_indices("she sells seashells", "She")
#     #=> [0, 13]
def find_word_indices(text, word)
  regex = Regexp.new("(?=" + word + ")", Regexp::IGNORECASE)
  text.enum_for(:scan, regex).map { Regexp.last_match.offset(0).first }
end


# Return a 'context' string about the given INDEX in the supplied TEXT.
# i.e. return the substring delimited by
#  [INDEX - CONTEXT_LIM, INDEX + CONTEXT_LIM].
# By default, CONTEXT_LIM = 50.
#
# Usage:
#   get_context_string("This is a long string of text", 5, 7)
#   #=> "This is a lon"
def get_context_string(text, index, context_lim=50)
  context_start = [0, index - context_lim].max
  context_end = [text.length - 1, index + context_lim].min 
  text[context_start..context_end]
end


# Search through the given ARCHIVES for each of the given
# KEYWORDS, and return a hash mapping each keyword to a list
# of the archives the keyword was found in (if any) and their
# surrounding contexts.
#
# Usage:
#   keywords_search(archives, ["keyword"])
#   #=> {"keyword"=>
#   #     {"http://archive.li/XXXXX"=>
#   #       ["The keyword appeared in this sentence"]}}
def keywords_search(archives, keywords)
  # For efficiency, we loop through the archives and collect all
  # of the keywords+context for each archive URLs at once. But this
  # isn't a nice way to structure the results, so we invert the
  # data structure afterward.
  archive_keywords = { }
  archives.each do |url, html|
    archive_keywords[url] = [ ]
    text = html.text
    keywords.each do |keyword|
      indices = find_word_indices(text, keyword)
      unless indices.empty?
        context = indices.map { |index| get_context_string(text, index) }
        archive_keywords[url] << {keyword => context}
      end
    end
  end

  # Invert the hash data structure so that we match keywords to
  # archives, instead of the other way around
  keywords = { }
  archive_keywords.each do |url, keywords_list|
    keywords_list.each do |keyword|
      keywords[keyword.keys[0]] ||= [ ]
      keywords[keyword.keys[0]] << {url=>keyword.values[0]}
    end
  end
  keywords
end


# Print the given SEARCH_RESULTS in a neat, pretty form where
# each keyword is followed by a list of archive URLs and the
# surrounding contexts in which the keyword appears.
def pretty_print_search_results(search_results)
  search_results.each do |keyword, appearances|
    puts "#{keyword}:"
    appearances.each do |appearance|
      url = appearance.keys[0]
      contexts = appearance.values[0]
      puts "    #{url}:"
      contexts.each do |context|
        # Remove any whitespace or newlines
        puts "        #{context.gsub(/\s+/, " ")}"
      end
    end
  end
end


# Assume the JSON file provided as first command line arg
if ARGV.length < 1
  raise(ArgumentError, "No JSON file provided.")
else
  json_file = ARGV[0]
  # And maybe do something with extra, optional command line args?
end

archive_urls = extract_archives_from_json(json_file)
archive_html = { }

# Download HTML for all archives (check: is this sane? Might
# take too long/send too many requests.)
archive_urls = archive_urls[0..9]  # DEBUG

archive_urls.each do |url|
  archive_html[url] = download_html_document_object(url)

  # Add in a 5s pause so that we don't trip archive.li's
  # "Are you a bot?" CAPTCHA
  sleep(5)
end


# Finally, instantiate an IRB REPL for interactive searching.
# Available variables: json_file, archive_urls, archive_html
# Available functions: download_html_document_object,
#   extract_archives_from_json, find_word_indices, get_context_string,
#   keywords_search, pretty_print_search_results
binding.irb