time-wasting-thread-memorial/scripts/scrape.moon

30 lines
687 B
Plaintext
Raw Normal View History

json = require "dkjson"
sqlite3 = require "lsqlite3"
db = assert sqlite3.open "../db/posts.db"
insert_stmt = assert db\prepare "INSERT INTO Posts VALUES (NULL, ?, ?, '', ?, ?)"
2015-07-28 02:17:43 +00:00
db\exec [[ BEGIN TRANSACTION; ]]
for page=1,8002
print "Scraping page #{page}..."
with fin = assert io.open "../raw/pages/#{page}.json", "r"
data = fin\read "*a"
2015-07-28 02:17:43 +00:00
posts, _, err = json.decode data
error err unless posts
for _, post in pairs posts.topics
2015-07-28 02:17:43 +00:00
continue unless post
do
insert_stmt\bind_values post.id, post.body, post.author, page
insert_stmt\step!
insert_stmt\reset!
fin\close!
2015-07-28 02:17:43 +00:00
print "done at #{os.date!}"
db\exec [[ COMMIT; ]]