Add simple RSS parser, currently only does planet. Use this for the

planet list on the frontpage and the community page.

diff --git a/pgweb/core/admin.py b/pgweb/core/admin.py
index 43381e47c428e56a149238171a29a749f4e5f0f0..c28b2ff08c68a049ab2488d1ca01802500ae20f3 100644 (file)
--- a/pgweb/core/admin.py
+++ b/pgweb/core/admin.py
@@ -14,4 +14,6 @@ class OrganisationAdmin(admin.ModelAdmin):
 admin.site.register(Version)
 admin.site.register(OrganisationType)
 admin.site.register(Organisation, OrganisationAdmin)
+admin.site.register(ImportedRSSFeed)
+admin.site.register(ImportedRSSItem)
 

diff --git a/pgweb/core/models.py b/pgweb/core/models.py
index 4ba02595b2b8323bc26b69f53a4064dbc32d1223..bf2d17337e691497ab59446df4910985428f960f 100644 (file)
--- a/pgweb/core/models.py
+++ b/pgweb/core/models.py
@@ -55,3 +55,25 @@ class Organisation(PgModel, models.Model):
        class Meta:
                ordering = ('name',)
 
+
+# Basic classes for importing external RSS feeds, such as planet
+class ImportedRSSFeed(models.Model):
+       internalname = models.CharField(max_length=32, null=False, blank=False, unique=True)
+       url = models.URLField(null=False, blank=False)
+
+       def __unicode__(self):
+               return self.internalname
+
+class ImportedRSSItem(models.Model):
+       feed = models.ForeignKey(ImportedRSSFeed)
+       title = models.CharField(max_length=100, null=False, blank=False)
+       url = models.URLField(null=False, blank=False)
+       posttime = models.DateTimeField(null=False, blank=False)
+
+       def __unicode__(self):
+               return self.title
+
+       @property
+       def date(self):
+               return self.posttime.strftime("%Y-%m-%d")
+

diff --git a/pgweb/core/views.py b/pgweb/core/views.py
index b6cf0d6e5a45efcc47b94e1dfb80b5e9b7a40736..5c475125f46b945db2c56c9df65f30ff31060213 100644 (file)
--- a/pgweb/core/views.py
+++ b/pgweb/core/views.py
@@ -11,7 +11,7 @@ from pgweb.util.helpers import simple_form
 from news.models import NewsArticle
 from events.models import Event
 from quotes.models import Quote
-from models import Version
+from models import Version, ImportedRSSFeed, ImportedRSSItem
 
 # models needed for the pieces on the community page
 from survey.models import Survey
@@ -26,6 +26,7 @@ def home(request):
        events = Event.objects.select_related('country').filter(approved=True).filter(training=False)[:3]
        quote = Quote.objects.filter(approved=True).order_by('?')[0]
        versions = Version.objects.all()
+       planet = ImportedRSSItem.objects.filter(feed__internalname="planet").order_by("-posttime")[:5]
 
        return render_to_response('index.html', {
                'title': 'The world\'s most advanced open source database',
@@ -33,6 +34,7 @@ def home(request):
                'events': events,
                'quote': quote,
                'versions': versions,
+               'planet': planet,
        })
 
 # Community main page (contains surveys and potentially more)
@@ -42,8 +44,10 @@ def community(request):
                s = s[0]
        except:
                s = None
+       planet = ImportedRSSItem.objects.filter(feed__internalname="planet").order_by("-posttime")[:7]
        return render_to_response('core/community.html', {
                'survey': s,
+               'planet': planet,
        }, NavContext(request, 'community'))
 
 # Generic fallback view for static pages

diff --git a/templates/core/community.html b/templates/core/community.html
index 39963ad57c2396179ca34b5fa44e43203580a2b9..a05c783694674713fce1ba561fe58b06c46c595e 100644 (file)
--- a/templates/core/community.html
+++ b/templates/core/community.html
@@ -33,21 +33,18 @@
 </div>
 {%endif%}
 
-<p>FIXME</p>
-<!-- BEGIN planet_block -->
 <div id="pgPlanetWrap">
 <div id="pgPlanet">
  <dl>
   <dt>Planet PostgreSQL</dt>
   <dd><ul>
-   <!-- BEGIN planet_loop -->
-    <li><a href="{link}">{title}</a></li>
-   <!-- END planet_loop -->
+   {%for post in planet%}
+    <li><a href="{{post.url}}">{{post.title}}</a></li>
+   {%endfor%}
   </ul></dd>
   <dd><a href="http://planet.postgresql.org">More</a></dd>
  </dl>
 </div>
 </div>
 
-<!-- END planet_block -->
 {%endblock%}

diff --git a/templates/index.html b/templates/index.html
index 72b336e603750e922dcf8278855fa76e97dc9427..099cc0ceaf49ee365eb757d3c3d6c9838045856c 100644 (file)
--- a/templates/index.html
+++ b/templates/index.html
@@ -119,12 +119,12 @@
            <img src="/media/img/hdr/hdr_planetpg.png" width="140" height="10" alt="Planet PostgreSQL" />
            </a>
            <div id="pgFrontPlanetWrap">
-            <!-- BEGIN planet_loop -->
+            {%for post in planet%}
             <p>
-              <span class="txtDate">{date}</span><br />
-              <a href="{link}">{title}</a>
+              <span class="txtDate">{{post.date}}</span><br />
+              <a href="{{post.url}}">{{post.title}}</a>
             </p>
-            <!-- END planet_loop -->
+            {%endfor%}
             <div>
              <img class="pgArrowImage" src="/layout/images/blt_blu_arrow.png" width="6" height="5" alt="" />
              <ul class="pgRSSBottomList">

diff --git a/tools/rss/fetch_rss_feeds.py b/tools/rss/fetch_rss_feeds.py
new file mode 100755 (executable)
index 0000000..a7afd5f
--- /dev/null
+++ b/tools/rss/fetch_rss_feeds.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import feedparser
+import psycopg2
+import socket
+
+
+# Set timeout for loading RSS feeds
+socket.setdefaulttimeout(20)
+
+
+db = psycopg2.connect('host=/tmp dbname=pgweb')
+curs = db.cursor()
+curs.execute("SELECT id,internalname,url FROM core_importedrssfeed")
+for id,internalname,url in curs.fetchall():
+       try:
+               feed = feedparser.parse(url)
+               
+               if not hasattr(feed, 'status'):
+                       # bozo_excpetion can seemingly be set when there is no error as well,
+                       # so make sure we only check if we didn't get a status.
+                       if hasattr(feed,'bozo_exception'):
+                               raise Exception('Feed load error %s' % feed.bozo_exception)
+                       raise Exception('Feed load error with no exception!')
+               if feed.status != 200:
+                       raise Exception('Feed returned status %s' % feed.status)
+               for entry in feed.entries:
+                       curs.execute("""INSERT INTO core_importedrssitem (feed_id, title, url, posttime)
+SELECT %(feed)s, %(title)s, %(url)s, %(posttime)s
+WHERE NOT EXISTS (SELECT * FROM core_importedrssitem c2 WHERE c2.feed_id=%(feed)s AND c2.url=%(url)s)""", {
+                               'feed': id,
+                               'title': entry.title,
+                               'url': entry.link,
+                               'posttime': entry.date,
+                       })
+       except Exception, e:
+               print "Failed to load %s: %s" % (internalname, e)
+
+db.commit()
+