Add scraper for @reednj's list
This commit is contained in:
parent
79ba928de3
commit
db7ecdbbfe
|
@ -0,0 +1,2 @@
|
||||||
|
nimcache
|
||||||
|
reednj
|
|
@ -0,0 +1,27 @@
|
||||||
|
import db_sqlite, future, htmlparser, httpclient, q, strtabs, xmltree, ../twtxt
|
||||||
|
|
||||||
|
let
|
||||||
|
reshtml = "http://twtxt.reednj.com/".getContent
|
||||||
|
doc = reshtml.q
|
||||||
|
users = doc.select "table tbody tr a.user-link"
|
||||||
|
feedurls = doc.select "table tbody tr td a.wide-only"
|
||||||
|
|
||||||
|
var
|
||||||
|
usernames = newSeq[string]()
|
||||||
|
feeds = newSeq[string]()
|
||||||
|
|
||||||
|
for user in users.items:
|
||||||
|
usernames.add user.innerText[1..^1]
|
||||||
|
|
||||||
|
for feed in feedurls.items:
|
||||||
|
feeds.add feed.attrs["href"]
|
||||||
|
|
||||||
|
let db = open("../../data/twtxt.db", nil, nil, nil)
|
||||||
|
|
||||||
|
for i, user in usernames.pairs:
|
||||||
|
let url = feeds[i]
|
||||||
|
echo i, " ", user, " ", url
|
||||||
|
|
||||||
|
try:
|
||||||
|
db.exec(sql"insert into users values(null, ?, ?)", user, url)
|
||||||
|
except: echo getCurrentExceptionMsg()
|
|
@ -9,7 +9,7 @@ bin = @["twtxtlist", "worker"]
|
||||||
|
|
||||||
# Dependencies
|
# Dependencies
|
||||||
|
|
||||||
requires "nim >= 0.13.0", "jester", "moustachu", "shorturl"
|
requires "nim >= 0.13.0", "jester", "moustachu", "shorturl", "q"
|
||||||
|
|
||||||
mode = ScriptMode.Verbose
|
mode = ScriptMode.Verbose
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue