oops

Ref https://github.com/buckket/twtxt/issues/3#issuecomment-182738604

src/templates/index.mustache

@@ -1,5 +1,7 @@
 <p>welcome to the twtxt list. this is an openly available list of users of <a href="https://github.com/buckket/twtxt">twtxt</a>, the decentralized microblogging tool. this will poll every submitted user every 5 minutes to update its view.</p>
 
+<p>this site currently indexes {{ users }} users and {{ tweets }} tweets.</p>
+
 <h3>links</h3>
 
 <ul>

src/twtxt.nim

@@ -1,7 +1,8 @@
 import db_sqlite, json, httpclient, os, strutils, times, time/gotime
 
 const
-  ISOTime*: string = "yyyy-MM-dd'T'HH:mm:ss'.000000+'zzz"
+  ISOTime*: string = "yyyy-MM-dd'T'HH:mm:ss'+'zzz"
+  version: string = staticExec "git describe --tags --long"
 
 type
   Tweet* = object of RootObj
@@ -49,11 +50,13 @@ proc fromDBRow*(r: Row): Tweet =
 proc getTweetsFrom*(url: string, username: string): seq[Tweet] =
   var res = newSeq[Tweet]()
 
-  let body = url.getContent(timeout=20_000)
+  let body = url.getContent(timeout=20_000, userAgent = "twtxtlist/" & version & " (+https://xena.greedo.xeserv.us/files/xena.txt; @xena)")
 
   for line in body.splitLines().items():
     if line.len > 0 and line[0] != '#':
-      res.add(username.parseTweet(line))
+      try:
+        res.add(username.parseTweet(line))
+      except: echo username & " " & getCurrentExceptionMsg()
 
   return res
 
@@ -67,14 +70,4 @@ proc updateTweetsByUser*(db: DBConn, username, url: string) {. gcsafe .} =
         db.exec(sql"insert into tweets values(null, ?, ?, ?);", username, tweet.date.timeInfoToTime().toSeconds().int(), tweet.message)
       except: discard
   except:
-    echo getCurrentExceptionMsg()
-
-proc updateTweetsOnce*() {. gcsafe .} =
-  var db = open("./data/twtxt.db", nil, nil, nil)
-
-  var users = db.getAllRows sql"select * from users where username != 'twtxtlist'"
-
-  for user in users.items():
-    echo "updating " & $user
-
-    db.updateTweetsByUser(user[1], user[2])
+    echo username & " " & getCurrentExceptionMsg()

src/twtxtlist.nim

@@ -1,4 +1,4 @@
-import asyncdispatch, db_sqlite, httpclient, jester, moustachu, os, shorturl,
+import asyncdispatch, db_sqlite, httpclient, jester, json, moustachu, os, shorturl,
   strutils, tables, templates/all, times, tweet/parse, twtxt, typetraits
 
 let
@@ -55,7 +55,15 @@ settings:
 
 routes:
   get "/":
-    renderMustache("home", indexTemplate, newContext())
+    var ctx = newContext()
+    let
+      users = db.getValue sql"select count(*) from users"
+      tweets = db.getValue sql"select count(*) from tweets"
+
+    ctx["users"] = users
+    ctx["tweets"] = tweets
+
+    renderMustache("home", indexTemplate, ctx)
 
   get "/changelog":
     const log = staticExec "git log -10"
@@ -93,6 +101,9 @@ routes:
 
       db.updateTweetsByUser(username, url)
       db.exec(sql"insert into users values (null, ?, ?)", username, url)
+      db.exec(sql"insert into tweets values (null, 'twtxtlist', ?, ?)",
+              $(getTime().getGMTime().timeinfoToTime().toSeconds()),
+              "Added user @<" & username & " " & url & ">")
 
       redirect "/users/" & username & "/0"
     except: fail()
@@ -114,6 +125,20 @@ routes:
 
     renderMustache "users", usersTemplate, ctx
 
+  get "/users.txt":
+    let
+      headers = {
+        "Content-Type": "text/plain"
+      }
+      users = db.getAllRows(sql"select username, url from users order by username")
+
+    var outBuf = ""
+
+    for user in users.items():
+      outBuf &= "twtxt follow " & user[0] & " " & user[1] & "\r\n"
+
+    resp Http200, headers, outBuf
+
   get "/users/@name/@page":
     try:
       var
@@ -160,7 +185,7 @@ routes:
   get "/timeline/@page":
     try:
       var
-        tweets = db.getAllRows(sql"select * from tweets order by time desc limit 20 offset ?", (@"page").parseInt() * 20)
+        tweets = db.getAllRows(sql"select * from tweets where username <> 'twtxtlist' and username <> 'directory' order by time desc limit 20 offset ?", (@"page").parseInt() * 20)
         ctx = newContext()
         tweetList = newSeq[Context]()
 

src/worker.nim

@@ -1,5 +1,15 @@
 import db_sqlite, os, streams, threadpool, twtxt
 
+proc updateTweetsOnce() {. gcsafe .} =
+  var db = open("./data/twtxt.db", nil, nil, nil)
+
+  var users = db.getAllRows sql"select * from users where username != 'twtxtlist'"
+
+  for user in users.items():
+    echo "updating " & $user
+
+    spawn db.updateTweetsByUser(user[1], user[2])
+
 proc updateSystemUser() =
   let
     fname = "public" / "twtxt.txt"