2016-02-07 20:48:43 +00:00
|
|
|
import db_sqlite, json, httpclient, os, strutils, times, time/gotime
|
2016-02-06 15:36:53 +00:00
|
|
|
|
|
|
|
const
|
2016-02-12 11:13:05 +00:00
|
|
|
ISOTime*: string = "yyyy-MM-dd'T'HH:mm:ss'+'zzz"
|
2016-02-11 11:00:58 +00:00
|
|
|
version: string = staticExec "git describe --tags --long"
|
2016-02-06 15:36:53 +00:00
|
|
|
|
|
|
|
type
|
|
|
|
Tweet* = object of RootObj
|
2016-02-12 19:37:31 +00:00
|
|
|
date*: TimeInfo
|
2016-02-06 15:36:53 +00:00
|
|
|
message*: string
|
|
|
|
username*: string
|
|
|
|
|
|
|
|
proc `$`*(t: Tweet): string =
|
2016-02-12 19:37:31 +00:00
|
|
|
return t.date.format(ISOTime) & " " & t.message
|
2016-02-06 15:36:53 +00:00
|
|
|
|
|
|
|
proc `%`*(t: Tweet): JsonNode =
|
|
|
|
%*
|
|
|
|
{
|
2016-02-12 19:37:31 +00:00
|
|
|
"date": t.date.format(ISOTime),
|
2016-02-06 15:36:53 +00:00
|
|
|
"message": t.message,
|
|
|
|
"username": t.username,
|
|
|
|
}
|
|
|
|
|
|
|
|
proc `%`*(s: seq[Tweet]): JsonNode =
|
|
|
|
result = newJArray()
|
|
|
|
|
|
|
|
for t in s.items():
|
|
|
|
result.add %t
|
|
|
|
|
|
|
|
proc parseTweet*(user, inp: string): Tweet =
|
|
|
|
let
|
|
|
|
splitTweet = inp.split '\t'
|
|
|
|
|
|
|
|
if splitTweet.len != 2:
|
2016-02-12 19:37:31 +00:00
|
|
|
raise newException(ValueError, "Invalid tweet")
|
2016-02-06 15:36:53 +00:00
|
|
|
|
|
|
|
let
|
2016-02-07 20:48:43 +00:00
|
|
|
tdate = splitTweet[0].parseTime
|
2016-02-06 15:36:53 +00:00
|
|
|
message = splitTweet[1]
|
|
|
|
|
|
|
|
Tweet(date: tdate,
|
|
|
|
message: message,
|
|
|
|
username: user)
|
|
|
|
|
2016-02-06 18:45:51 +00:00
|
|
|
proc fromDBRow*(r: Row): Tweet =
|
2016-02-12 19:37:31 +00:00
|
|
|
Tweet(date: r[2].split(".")[0].parseInt().fromSeconds().timeToTimeInfo(),
|
2016-02-06 18:45:51 +00:00
|
|
|
username: r[1],
|
|
|
|
message: r[3])
|
|
|
|
|
2016-02-06 15:36:53 +00:00
|
|
|
proc getTweetsFrom*(url: string, username: string): seq[Tweet] =
|
|
|
|
var res = newSeq[Tweet]()
|
|
|
|
|
2016-02-11 12:57:52 +00:00
|
|
|
let body = url.getContent(timeout=20_000, userAgent = "twtxtlist/" & version & " (+https://xena.greedo.xeserv.us/files/xena.txt; @xena)")
|
2016-02-06 15:36:53 +00:00
|
|
|
|
|
|
|
for line in body.splitLines().items():
|
2016-02-09 17:29:21 +00:00
|
|
|
if line.len > 0 and line[0] != '#':
|
2016-02-12 10:18:17 +00:00
|
|
|
try:
|
|
|
|
res.add(username.parseTweet(line))
|
|
|
|
except: echo username & " " & getCurrentExceptionMsg()
|
2016-02-06 15:36:53 +00:00
|
|
|
|
|
|
|
return res
|
|
|
|
|
2016-02-10 18:50:01 +00:00
|
|
|
proc updateTweetsByUser*(db: DBConn, username, url: string) {. gcsafe .} =
|
|
|
|
try:
|
|
|
|
let
|
|
|
|
tweets = getTweetsFrom(url, username)
|
|
|
|
|
|
|
|
for tweet in tweets.items():
|
|
|
|
try:
|
2016-02-12 19:37:31 +00:00
|
|
|
db.exec(sql"insert into tweets values(null, ?, ?, ?);", username, tweet.date.timeInfoToTime().toSeconds().int(), tweet.message)
|
2016-02-10 18:50:01 +00:00
|
|
|
except: discard
|
|
|
|
except:
|
2016-02-12 10:18:17 +00:00
|
|
|
echo username & " " & getCurrentExceptionMsg()
|