24 lines
553 B
Plaintext
24 lines
553 B
Plaintext
json = require "dkjson"
|
|
sqlite3 = require "lsqlite3"
|
|
|
|
db = assert sqlite3.open "../db/posts.db"
|
|
|
|
insert_stmt = assert db\prepare "INSERT INTO Posts VALUES (NULL, ?, ?, '', ?, ?)"
|
|
|
|
for page=1,8002
|
|
print "Scraping page #{page}..."
|
|
|
|
with fin = assert io.open "../raw/pages/#{page}.json", "r"
|
|
data = fin\read "*a"
|
|
posts = json.decode data
|
|
|
|
for _, post in pairs posts.topics
|
|
do
|
|
insert_stmt\bind_values post.id, post.body, post.author, page
|
|
insert_stmt\step!
|
|
insert_stmt\reset!
|
|
|
|
fin\close!
|
|
|
|
print "done"
|