Add scraper for @reednj's list
This commit is contained in:
parent
79ba928de3
commit
db7ecdbbfe
|
@ -0,0 +1,2 @@
|
|||
nimcache
|
||||
reednj
|
|
@ -0,0 +1,27 @@
|
|||
import db_sqlite, future, htmlparser, httpclient, q, strtabs, xmltree, ../twtxt
|
||||
|
||||
let
|
||||
reshtml = "http://twtxt.reednj.com/".getContent
|
||||
doc = reshtml.q
|
||||
users = doc.select "table tbody tr a.user-link"
|
||||
feedurls = doc.select "table tbody tr td a.wide-only"
|
||||
|
||||
var
|
||||
usernames = newSeq[string]()
|
||||
feeds = newSeq[string]()
|
||||
|
||||
for user in users.items:
|
||||
usernames.add user.innerText[1..^1]
|
||||
|
||||
for feed in feedurls.items:
|
||||
feeds.add feed.attrs["href"]
|
||||
|
||||
let db = open("../../data/twtxt.db", nil, nil, nil)
|
||||
|
||||
for i, user in usernames.pairs:
|
||||
let url = feeds[i]
|
||||
echo i, " ", user, " ", url
|
||||
|
||||
try:
|
||||
db.exec(sql"insert into users values(null, ?, ?)", user, url)
|
||||
except: echo getCurrentExceptionMsg()
|
|
@ -9,7 +9,7 @@ bin = @["twtxtlist", "worker"]
|
|||
|
||||
# Dependencies
|
||||
|
||||
requires "nim >= 0.13.0", "jester", "moustachu", "shorturl"
|
||||
requires "nim >= 0.13.0", "jester", "moustachu", "shorturl", "q"
|
||||
|
||||
mode = ScriptMode.Verbose
|
||||
|
||||
|
|
Loading…
Reference in New Issue