Description: improve TerraParser for a better and more accurate search.
Author: Leo Iannacone <l3on@ubuntu.com>
Bug-Ubuntu: https://bugs.launchpad.net/bugs/1062356

---
 plugins/lyrics/TerraParser.py |   82 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 14 deletions(-)

--- a/plugins/lyrics/TerraParser.py
+++ b/plugins/lyrics/TerraParser.py
@@ -1,6 +1,7 @@
 # -*- Mode: python; coding: utf-8; tab-width: 8; indent-tabs-mode: t; -*-
 #
 # Copyright (C) 2009 Hardy Beltran Monasterios
+#               2012 Leo Iannacone
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,6 +31,7 @@
 import rb
 import re
 import sys
+import stringmatch
 
 # Deal with html entitys and utf-8
 # code taken from django/utils/text.py
@@ -38,6 +40,8 @@
 
 pattern = re.compile("&(#?\w+?);")
 
+min_song_match = .5
+
 def _replace_entity(match):
 	text = match.group(1)
 	if text[0] == u'#':
@@ -63,9 +67,13 @@
 	def __init__(self, artist, title):
 		self.artist = artist
 		self.title = title
+		self.div_search = '<div id=[\'|"]letra|div_letra[\'|"]>'
+		self.div_accurate_search = ''
+		self.base_path = 'http://letras.mus.br'
+		self.more_accurate = False
 
 	def search(self, callback, *data):
-		path = 'http://letras.terra.com.br/'
+		path = self.base_path
 
 		artist = urllib.quote(self.artist)
 		title = urllib.quote(self.title)
@@ -75,7 +83,7 @@
 		print "search URL: " + wurl
 
 		loader = rb.Loader()
-		loader.get_url (path + wurl, self.got_lyrics, callback, *data)
+		loader.get_url ("%s/%s" % (path, wurl), self.got_lyrics, callback, *data)
 
 	def got_lyrics(self, result, callback, *data):
 		if result is None:
@@ -83,30 +91,76 @@
 			return
 
 		if result is not None:
-			result = result.decode('iso-8859-1').encode('UTF-8')
 			if re.search('M&uacute;sica n&atilde;o encontrada', result):
 				print "not found"
 				callback (None, *data)
-			elif re.search('<div id="letra">', result):
-				callback(self.parse_lyrics(result), *data)
+			elif re.search(self.div_search, result):
+				if re.search('Confira abaixo a lista completa do artist', result):
+					self.get_moreaccurate_lyrics(result, callback, *data)
+				else:
+					callback(self.parse_lyrics(result), *data)
 			else:
 				callback (None, *data)
+				return
 		else:
 			callback (None, *data)
+			return
 
+	def get_moreaccurate_lyrics(self, source, callback, *data):
+		artist = urllib.quote(self.artist)
+		links_section = re.split(self.div_search, source)[1]
+		pattern_link =  '<a href=[\'|"]%s/%s' % (self.base_path, self.artist)
+		links = re.split (pattern_link, links_section.lower())
+		if not links > 2:
+			callback (None, *data)
+			return
+		links.pop(0)
+		links.pop(0)
+		pattern_song = '([^/].*)/(.*)>(.*)<small'
+		smvalue_bestmatch = 0
+		url_bestmatch = ""
+
+		for link in links:
+			info = re.findall(pattern_song, link)[0]
+			if (len(info) < 3):
+				continue
+			url = info[0]
+			title = info[2]
+			smvalue = stringmatch.string_match (title.lower().replace(' ', '' ),
+					   self.title.lower().replace(' ', ''))
+			if smvalue > min_song_match and smvalue > smvalue_bestmatch:
+				url_bestmatch = url
+				smvalue_bestmatch = smvalue
+				if smvalue == 1:
+					break
+		if url_bestmatch:
+			print "best match: %s %s %s" % (smvalue_bestmatch, artist, url_bestmatch) 
+			self.more_accurate = True
+			loader = rb.Loader()
+			loader.get_url ("%s/%s/%s" % (self.base_path, artist, url_bestmatch), self.got_lyrics, callback, *data)
+		else:
+			callback (None, *data)
+			return
+		
 
 	def parse_lyrics(self, source):
-		source = re.split('<div id="letra">', source)[1]
-		source = re.split('<p>', source)
-		# Parse artist and title
-		artistitle = re.sub('<.*?>', '', source[0])
-		rx = re.compile('^(\t|\n)+',re.M | re.S)
-		artistitle = rx.sub('', artistitle)
+		source = source.decode('iso-8859-1').encode('UTF-8')
+		# Replace some splecial char:
+		source = re.sub('\xc2\x92', "'", source)
+		source = re.split(self.div_search, source)[1]
+		lyrics = ""
 		# Parse lyrics
-		lyrics = re.split('</p>', source[1])[0]
+		if not self.more_accurate:
+			source = re.split('<p>', source)
+			lyrics = re.split('</p>', source[1])[0]
+		else:
+			lyrics = re.split('</div>', source)[0]
+			lyrics = re.sub('<[pP]>', '', lyrics)
+			lyrics = re.sub('</[pP]>', '\n', lyrics)
+			
 		lyrics = re.sub('<[Bb][Rr]/>', '', lyrics)
 
-		lyrics = unescape_entities(artistitle) + "\n" + unescape_entities(lyrics)
-		lyrics += "\n\nEsta letra foi disponibilizada pelo site\nhttp://letras.terra.com.br"
+		lyrics = unescape_entities(lyrics)
+		lyrics += "\n\nEsta letra foi disponibilizada pelo site\n%s" % self.base_path
 
 		return lyrics
