time-wasting-thread-memorial/scripts/scrape.moon

json    = require "dkjson"
sqlite3 = require "lsqlite3"

db = assert sqlite3.open "../db/posts.db"

insert_stmt = assert db\prepare "INSERT INTO Posts VALUES (NULL, ?, ?, '', ?, ?)"

db\exec [[ BEGIN TRANSACTION; ]]

for page=1,8002
  print "Scraping page #{page}..."

  with fin = assert io.open "../raw/pages/#{page}.json", "r"
    data = fin\read "*a"
    posts, _, err = json.decode data
    error err unless posts

    for _, post in pairs posts.topics
      continue unless post
      do
        insert_stmt\bind_values post.id, post.body, post.author, page
        insert_stmt\step!
        insert_stmt\reset!

    fin\close!

  print "done at #{os.date!}"

db\exec [[ COMMIT; ]]