make wolframalpha and urban use apis instead of scraping

plugins/dictionary.py

@@ -8,16 +8,14 @@ from util import hook, http
 def urban(inp):
     '''.u/.urban <phrase> -- looks up <phrase> on urbandictionary.com'''
 
-    url = 'http://www.urbandictionary.com/define.php'
-    page = http.get_html(url, term=inp)
-    words = page.xpath("//td[@class='word']")
-    defs = page.xpath("//div[@class='definition']")
+    url = 'http://www.urbandictionary.com/iphone/search/define'
+    page = http.get_json(url, term=inp)
+    defs = page['list']
 
-    if not defs:
-        return 'no definitions found'
+    if page['result_type'] == 'no_results':
+        return 'not found.'
 
-    out = words[0].text_content().strip() + ': ' + ' '.join(
-            defs[0].text_content().split())
+    out = defs[0]['word'] + ': ' + defs[0]['definition']
 
     if len(out) > 400:
         out = out[:out.rfind(' ', 0, 400)] + '...'

plugins/wolframalpha.py

@@ -5,34 +5,31 @@ from util import hook, http
 
 @hook.command('wa')
 @hook.command
-def wolframalpha(inp):
-    ".wa/.wolframalpha <query> -- scrapes Wolfram Alpha's" \
-            " results for <query>"
+def wolframalpha(inp, bot=None):
+    ".wa/.wolframalpha <query> -- computes <query> using Wolfram Alpha"
 
-    url = "http://www.wolframalpha.com/input/?asynchronous=false"
+    api_key = bot.config.get("api_keys", {}).get("wolframalpha", None)
+    if api_key is None:
+        return "error: no api key set"
 
-    h = http.get_html(url, i=inp)
+    url = 'http://api.wolframalpha.com/v2/query?format=plaintext'
 
-    pods = h.xpath("//div[@class='pod ']")
+    result = http.get_xml(url, input=inp, appid=api_key)
 
     pod_texts = []
-    for pod in pods:
-        heading = pod.find('h2')
-        if heading is not None:
-            heading = heading.text_content().strip()
-            if heading.startswith('Input'):
-                continue
-        else:
+    for pod in result.xpath("//pod"):
+        title = pod.attrib['title']
+        if pod.attrib['id'] == 'Input':
             continue
 
         results = []
-        for alt in pod.xpath('div/div[@class="output pnt"]/img/@alt'):
-            alt = alt.strip().replace('\\n', '; ')
-            alt = re.sub(r'\s+', ' ', alt)
-            if alt:
-                results.append(alt)
+        for subpod in pod.xpath('subpod/plaintext/text()'):
+            subpod = subpod.strip().replace('\\n', '; ')
+            subpod = re.sub(r'\s+', ' ', subpod)
+            if subpod:
+                results.append(subpod)
         if results:
-            pod_texts.append(heading + ' ' + '|'.join(results))
+            pod_texts.append(title + ': ' + '|'.join(results))
 
     ret = '. '.join(pod_texts)
 
@@ -51,6 +48,6 @@ def wolframalpha(inp):
         ret = re.sub(r'\W+$', '', ret) + '...'
 
     if not ret:
-        return 'no result'
+        return 'no results'
 
     return ret