• R/O
  • SSH
  • HTTPS

pyhabu: Commit


Commit MetaInfo

Revision93 (tree)
Time2008-01-09 12:17:24
Authorliris

Log Message

plugin is moved from pyhabu-plugin project

Change Summary

Incremental Difference

--- trunk/contrib/plugin/filter/html_to_feed.py (nonexistent)
+++ trunk/contrib/plugin/filter/html_to_feed.py (revision 93)
@@ -0,0 +1,97 @@
1+__version__ = 'Alpha'
2+
3+# original code: filter.custom_feed in pyhabu-plugin
4+
5+import re
6+import sets
7+from StringIO import StringIO
8+
9+from twisted.internet import reactor
10+from twisted.internet.defer import Deferred
11+from lxml import etree
12+
13+import habu.log as log
14+from habu.habuutils import toUTF8
15+from habu.webutils import getPage
16+
17+
18+class CustomFeed(object):
19+ def __init__(self, config, environ):
20+ self.config = config
21+ self.proxy_host = environ.get("proxy_host", None)
22+ self.proxy_port = environ.get("proxy_port", 0)
23+ self.maxThread = config.setdefault("max-thread", 0)
24+
25+ def execute(self, context):
26+ feed = [toUTF8(entry.get("link")) for entry in context["entries"]]
27+ feed = sets.Set(feed)
28+ try:
29+ feed.remove("")
30+ except KeyError:
31+ pass
32+
33+ deferredList = [Deferred() for f in range(0, len(feed))]
34+
35+ maxThread = min(
36+ self.maxThread > 0 and self.maxThread or len(feed),
37+ len(feed))
38+
39+ for i in range(0, maxThread):
40+ self.run(feeds, deferredList)
41+
42+ return deferredList
43+
44+ @defer.deferredGenerator
45+ def run(self, feeds, deferredList):
46+ while feeds:
47+ url = feeds.pop()
48+ try:
49+ deferred = deferredList.pop()
50+ wfd = defer.waitForDeferred(
51+ getPage(url, self.proxy_host, self.proxy_port))
52+ yield wfd
53+ result = wfd.getResult()
54+ content = self.gotPage(result, url)
55+ deferred.callback(dict(entries=items))
56+ except Execption, e:
57+ log.debug("filter.config gotErr")
58+ deferred.errback(failure)
59+
60+
61+ def gotPage(self, content, url):
62+ body = None
63+ capture = self.config.get("capture", None)
64+ if capture:
65+ matched = re.findall(capture, content, re.DOTALL|re.IGNORECASE)
66+ if matched:
67+ body = matched[0]
68+ if not body:
69+ body = content
70+ parser = etree.HTMLParser()
71+
72+ parsedContent = etree.parse(StringIO(content), parser)
73+ try:
74+ charset = parsedContent.xpath('/html/head/meta[@http-equiv="Content-Type"]')[0].attrib.get("content")
75+ except IndexError:
76+ charset = None
77+ if charset:
78+ charset = charset.split("charset=")[-1].lower()
79+ title = parsedContent.xpath('/html/head/title')[0].text
80+
81+ items = list()
82+ items.append(dict(
83+ summary_detail = dict(
84+ base = "",
85+ type = "text/html",
86+ value = unicode(body, charset or "utf-8", "ignore"),
87+ language = "",
88+ ),
89+ title = title,
90+ link = url,
91+ ))
92+ return dict(entries=items)
93+
94+
95+def create(config, environ):
96+ return CustomFeed(config, environ)
97+
--- trunk/contrib/plugin/filter/exec_format.py (nonexistent)
+++ trunk/contrib/plugin/filter/exec_format.py (revision 93)
@@ -0,0 +1,73 @@
1+"""filter.docstring
2+
3+"""
4+import unittest
5+from StringIO import StringIO
6+
7+import habu.log as log
8+
9+
10+__version__ = "Alpha"
11+MODNAME = "filter.docstring"
12+
13+
14+class Main(object):
15+ def __init__(self, config, environ):
16+ self.config = config
17+ self.environ = environ
18+ self.ignore = config.get("ignore", False)
19+ template = config.get("template")
20+ if template:
21+ try:
22+ template = open(template).read()
23+ except IOError:
24+ log.debug("%s: IOError in template." % MODNAME)
25+ if not template:
26+ template = "NON-TEMPLATE"
27+ self.template = template
28+
29+ def execute(self, context):
30+ context["entries"] = [self.render(entry) for entry in context["entries"]]
31+ return context
32+
33+ def render(self, entry, template=None):
34+ """
35+ >>> template = "%(__version__)s/%(__doc__)s/%(__author__)s"
36+ >>> entry = dict(
37+ ... summary = '''
38+ ... \"\"\"SUMMARY.\"\"\"
39+ ... __author__ = "AUTHOR"
40+ ... __version__ = "VERSION"
41+ ... ''',
42+ ... )
43+ >>> create(dict(), dict()).render(entry, template)
44+ {'summary': 'VERSION/SUMMARY./AUTHOR'}
45+ """
46+ template = template or self.template
47+ mod = dict()
48+ try:
49+ exec entry["summary"] in mod
50+ except:
51+ if self.ignore:
52+ log.debug("%s: except ignore in %s." %
53+ (MODNAME, entry["title"]))
54+ else:
55+ raise
56+ else:
57+ entry["summary"] = template % mod
58+ return entry
59+
60+
61+def create(*argv, **kwargv):
62+ return Main(*argv, **kwargv)
63+
64+
65+class Test(unittest.TestCase):
66+ def test(self):
67+ pass
68+
69+
70+if __name__ == "__main__":
71+ import doctest
72+ doctest.testmod()
73+ unittest.main()
--- trunk/contrib/plugin/publisher/twitterPost.py (nonexistent)
+++ trunk/contrib/plugin/publisher/twitterPost.py (revision 93)
@@ -0,0 +1,50 @@
1+# -*- coding: utf-8 -*-
2+__author__ = 'mattn.jp@gmail.com'
3+__version__ = '0.1'
4+
5+from twitter import Api
6+import habu.log as log
7+
8+class TwitterPublisher(object):
9+ def __init__(self, config, environ):
10+ self.username = config.get("username", "twitter username")
11+ self.password = config.get("password", "twitter password")
12+
13+ def success(self, result):
14+ log.info("twitterPost : commit")
15+ return None
16+
17+ def gotErr(self, failure):
18+ log.error(failure)
19+ return None
20+
21+ def postTwitter(self, messages):
22+ api = Api(self.username, self.password)
23+ for msg in messages:
24+ api.PostUpdate(msg)
25+ return True
26+
27+ def execute(self, content):
28+ # prepare message
29+ messages = []
30+ for entry in content["entries"]:
31+ msg = entry["title"] + ":"
32+ if len(entry["summary"]):
33+ msg += entry["summary"]
34+ else:
35+ msg += entry["description"]
36+ msg += " " + entry["link"]
37+ if len(msg) > 159:
38+ msg = msg[0: 159] + "..."
39+ messages.append(msg)
40+
41+ # update twitter by thread.
42+ deferred = Deferred()
43+ threads.deferToThread(self.postTwitter, messages).addCallback(
44+ self.success, deferred).addErrback(
45+ self.gotErr, deferred)
46+
47+ return content
48+
49+def create(config, environ):
50+ return TwitterPublisher(config, environ)
Show on old repository browser