Ubuntu Software Center

Merge lp:~mvo/software-center/de-duplication-multiple-pkgnames-lp1043159 into lp:software-center

  1. de-duplication-multiple-pkgnames-lp1043159
  2. Merge into trunk
Proposed by Michael Vogt
Status: Merged
Merged at revision: 3210
Proposed branch: lp:~mvo/software-center/de-duplication-multiple-pkgnames-lp1043159
Merge into: lp:software-center
Prerequisite: lp:~mvo/software-center/exact-match-duplication-lp891613
Diff against target: 230 lines (+57/-38)
6 files modified
softwarecenter/db/categories.py (+3/-4)
softwarecenter/db/database.py (+24/-3)
softwarecenter/db/enquire.py (+1/-1)
softwarecenter/db/utils.py (+0/-14)
softwarecenter/ui/gtk3/views/lobbyview.py (+1/-2)
tests/test_database.py (+28/-14)
To merge this branch: bzr merge lp:~mvo/software-center/de-duplication-multiple-pkgnames-lp1043159
Reviewer Review Type Date Requested Status
Gary Lasker (community) Approve
Review via email: mp+126937@code.launchpad.net

This proposal supersedes a proposal from 2012-09-28.

Description of the change

This branch ensures that duplicatied packages are eliminated from the
get_query_for_pkgnames() if there are multiple documents in the database.
This can happen when a pacakge is available via the apt-xapian-index and
also via the software-center-agent (e.g. packages from extras.ubuntu.com)

To post a comment you must log in.
Revision history for this message
Gary Lasker (gary-lasker) wrote :

Hi Michael, thanks for this branch. Please see my comment in MP https://code.launchpad.net/~mvo/software-center/exact-match-duplication-lp891613/+merge/126902 that describes a strange side-effect that I noticed. It occurs when this branch is merged also. Thanks!

Revision history for this message
Gary Lasker (gary-lasker) wrote :

Hi Michael, when running this branch I'm hitting two exceptions, one at startup as shown:

Traceback (most recent call last):
  File "/home/tremolux/Projects/quantal/software-center_de-duplication_multiple_pkgnames_lp1043159_tweak/software-center/softwarecenter/db/categories.py", line 188, in _recommend_me_result
    self.query = self.db.get_query_for_pkgnames(pkgs)
AttributeError: 'RecommendedForYouCategory' object has no attribute 'db'

And the other when viewing the app details view:

Traceback (most recent call last):
  File "/home/tremolux/Projects/quantal/software-center_de-duplication_multiple_pkgnames_lp1043159_tweak/software-center/softwarecenter/db/categories.py", line 230, in _recommend_app_result
    self.query = self.db.get_query_for_pkgnames(pkgs)
AttributeError: 'AppRecommendationsCategory' object has no attribute 'db'

It seems we are just missing passing the db into a couple of classes. I made a small branch that should take care of these issues, and also updates the unit tests per the changes. Please check it over and feel free to use it if it looks right to you:

  lp:~gary-lasker/software-center/de-duplication-multiple-pkgnames-lp1043159-tweak

I will approve the branch in any case, please feel free to merge with the needed fix (or I can do it tomorrow).

Many thanks!

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'softwarecenter/db/categories.py'
--- softwarecenter/db/categories.py 2012-09-27 13:02:26 +0000
+++ softwarecenter/db/categories.py 2012-09-28 11:48:24 +0000
@@ -42,7 +42,6 @@
42from softwarecenter.backend.recagent import RecommenderAgent42from softwarecenter.backend.recagent import RecommenderAgent
43from softwarecenter.db.appfilter import AppFilter43from softwarecenter.db.appfilter import AppFilter
44from softwarecenter.db.enquire import AppEnquire44from softwarecenter.db.enquire import AppEnquire
45from softwarecenter.db.utils import get_query_for_pkgnames
46from softwarecenter.region import get_region_cached45from softwarecenter.region import get_region_cached
47from softwarecenter.utils import utf846from softwarecenter.utils import utf8
4847
@@ -183,10 +182,10 @@
183 pkgs.append(item['package_name'])182 pkgs.append(item['package_name'])
184 if self.subcategory:183 if self.subcategory:
185 self.query = xapian.Query(xapian.Query.OP_AND,184 self.query = xapian.Query(xapian.Query.OP_AND,
186 get_query_for_pkgnames(pkgs),185 self.db.get_query_for_pkgnames(pkgs),
187 self.subcategory.query)186 self.subcategory.query)
188 else:187 else:
189 self.query = get_query_for_pkgnames(pkgs)188 self.query = self.db.get_query_for_pkgnames(pkgs)
190 self.emit("needs-refresh")189 self.emit("needs-refresh")
191190
192 def _recommender_agent_error(self, recommender_agent, msg):191 def _recommender_agent_error(self, recommender_agent, msg):
@@ -227,7 +226,7 @@
227 pkgs = []226 pkgs = []
228 for item in result_list['data']:227 for item in result_list['data']:
229 pkgs.append(item['package_name'])228 pkgs.append(item['package_name'])
230 self.query = get_query_for_pkgnames(pkgs)229 self.query = self.db.get_query_for_pkgnames(pkgs)
231 self.emit("needs-refresh")230 self.emit("needs-refresh")
232231
233 def _recommender_agent_error(self, recommender_agent, msg):232 def _recommender_agent_error(self, recommender_agent, msg):
234233
=== modified file 'softwarecenter/db/database.py'
--- softwarecenter/db/database.py 2012-09-14 13:40:18 +0000
+++ softwarecenter/db/database.py 2012-09-28 11:48:24 +0000
@@ -24,13 +24,12 @@
24import threading24import threading
25import xapian25import xapian
26from softwarecenter.db.application import Application26from softwarecenter.db.application import Application
27from softwarecenter.db.utils import get_query_for_pkgnames
28from softwarecenter.db.pkginfo import get_pkg_info27from softwarecenter.db.pkginfo import get_pkg_info
29import softwarecenter.paths28import softwarecenter.paths
3029
31from gi.repository import GObject, Gio30from gi.repository import GObject, Gio
3231
33#from softwarecenter.utils import *32from softwarecenter.utils import ExecutionTime
34from softwarecenter.enums import (33from softwarecenter.enums import (
35 AVAILABLE_FOR_PURCHASE_MAGIC_CHANNEL_NAME,34 AVAILABLE_FOR_PURCHASE_MAGIC_CHANNEL_NAME,
36 PkgStates,35 PkgStates,
@@ -302,6 +301,28 @@
302 assert popcon_max > 0301 assert popcon_max > 0
303 return popcon_max302 return popcon_max
304303
304 def get_query_for_pkgnames(self, pkgnames):
305 """ return a xapian query that matches exactly the list of pkgnames """
306 enquire = xapian.Enquire(self.xapiandb)
307 query = xapian.Query()
308 for pkgname in pkgnames:
309 # even on the raspbery pi this query super quick (~0.003s)
310 with ExecutionTime("de-dup query_for_pkgnames"):
311 tmp_query = xapian.Query("AP" + pkgname)
312 enquire.set_query(tmp_query)
313 result = enquire.get_mset(0, 1)
314 # see bug #1043159, we need to ensure that we de-duplicate
315 # when there is a pkg and a app (e.g. from the s-c-agent) in the db
316 if len(result) == 1:
317 query = xapian.Query(xapian.Query.OP_OR,
318 query,
319 xapian.Query("AP" + pkgname))
320 else:
321 query = xapian.Query(xapian.Query.OP_OR,
322 query,
323 xapian.Query("XP" + pkgname))
324 return query
325
305 def get_query_list_from_search_entry(self, search_term,326 def get_query_list_from_search_entry(self, search_term,
306 category_query=None):327 category_query=None):
307 """ get xapian.Query from a search term string and a limit the328 """ get xapian.Query from a search term string and a limit the
@@ -343,7 +364,7 @@
343 search_term = search_term.strip()364 search_term = search_term.strip()
344 # get a pkg query365 # get a pkg query
345 if "," in search_term:366 if "," in search_term:
346 pkg_query = get_query_for_pkgnames(search_term.split(","))367 pkg_query = self.get_query_for_pkgnames(search_term.split(","))
347 else:368 else:
348 pkg_query = xapian.Query()369 pkg_query = xapian.Query()
349 for term in search_term.split():370 for term in search_term.split():
350371
=== modified file 'softwarecenter/db/enquire.py'
--- softwarecenter/db/enquire.py 2012-09-28 11:48:24 +0000
+++ softwarecenter/db/enquire.py 2012-09-28 11:48:24 +0000
@@ -160,7 +160,7 @@
160 if exact_pkgname_query:160 if exact_pkgname_query:
161 with ExecutionTime("de-duplication"):161 with ExecutionTime("de-duplication"):
162 q_app = xapian.Query(terms[0].replace("XP", "AP"))162 q_app = xapian.Query(terms[0].replace("XP", "AP"))
163 nr_apps, nr_pkgs = self._get_estimate_nr_apps_and_nr_pkgs(163 nr_apps, nr_pkgs = self._get_estimate_nr_apps_and_nr_pkgs(
164 enquire, q_app, xfilter)164 enquire, q_app, xfilter)
165 if nr_apps == 1:165 if nr_apps == 1:
166 q = q_app166 q = q_app
167167
=== modified file 'softwarecenter/db/utils.py'
--- softwarecenter/db/utils.py 2012-09-14 07:12:33 +0000
+++ softwarecenter/db/utils.py 2012-09-28 11:48:24 +0000
@@ -18,7 +18,6 @@
1818
19import logging19import logging
20import os20import os
21import xapian
2221
23from gi.repository import GObject22from gi.repository import GObject
2423
@@ -45,19 +44,6 @@
45 GObject.child_watch_add(pid, _on_update_software_center_agent_finished)44 GObject.child_watch_add(pid, _on_update_software_center_agent_finished)
4645
4746
48def get_query_for_pkgnames(pkgnames):
49 """ return a xapian query that matches exactly the list of pkgnames """
50 query = xapian.Query()
51 for pkgname in pkgnames:
52 query = xapian.Query(xapian.Query.OP_OR,
53 query,
54 xapian.Query("XP" + pkgname))
55 query = xapian.Query(xapian.Query.OP_OR,
56 query,
57 xapian.Query("AP" + pkgname))
58 return query
59
60
61def get_installed_apps_list(db):47def get_installed_apps_list(db):
62 """ return a list of installed applications """48 """ return a list of installed applications """
63 apps = set()49 apps = set()
6450
=== modified file 'softwarecenter/ui/gtk3/views/lobbyview.py'
--- softwarecenter/ui/gtk3/views/lobbyview.py 2012-09-24 09:49:23 +0000
+++ softwarecenter/ui/gtk3/views/lobbyview.py 2012-09-28 11:48:24 +0000
@@ -46,7 +46,6 @@
46 CategoriesParser,46 CategoriesParser,
47 get_category_by_name,47 get_category_by_name,
48 categories_sorted_by_name)48 categories_sorted_by_name)
49from softwarecenter.db.utils import get_query_for_pkgnames
50from softwarecenter.distro import get_distro49from softwarecenter.distro import get_distro
51from softwarecenter.backend.scagent import SoftwareCenterAgent50from softwarecenter.backend.scagent import SoftwareCenterAgent
52from softwarecenter.backend.reviews import get_review_loader51from softwarecenter.backend.reviews import get_review_loader
@@ -131,7 +130,7 @@
131 app = Application("", pkgs[0])130 app = Application("", pkgs[0])
132 self.emit("application-activated", app)131 self.emit("application-activated", app)
133 else:132 else:
134 query = get_query_for_pkgnames(pkgs)133 query = self.db.get_query_for_pkgnames(pkgs)
135 title = exhibit.title_translated134 title = exhibit.title_translated
136 untranslated_name = exhibit.package_names135 untranslated_name = exhibit.package_names
137 # create a temp query136 # create a temp query
138137
=== modified file 'tests/test_database.py'
--- tests/test_database.py 2012-09-28 11:48:24 +0000
+++ tests/test_database.py 2012-09-28 11:48:24 +0000
@@ -45,8 +45,9 @@
45 get_installed_apps_list,45 get_installed_apps_list,
46)46)
47from softwarecenter.enums import (47from softwarecenter.enums import (
48 NonAppVisibility,
49 PkgStates,
48 XapianValues,50 XapianValues,
49 PkgStates,
50 )51 )
51from softwarecenter.region import (52from softwarecenter.region import (
52 REGION_BLACKLIST_TAG,53 REGION_BLACKLIST_TAG,
@@ -866,32 +867,45 @@
866]867]
867"""868"""
868869
869 def test_search_app_pkgname_duplication_lp891613(self):870 @classmethod
870 # create a fake database to simualte a run of software-center-agent871 def setUpClass(cls):
871 cache = get_pkg_info()872 cache = get_pkg_info()
872 cache.open()873 cache.open()
873 db = xapian.WritableDatabase(TEST_DB,874 db = xapian.WritableDatabase(TEST_DB,
874 xapian.DB_CREATE_OR_OVERWRITE)875 xapian.DB_CREATE_OR_OVERWRITE)
875 res = update_from_json_string(876 update_from_json_string(db, cache, cls.APP_INFO_JSON, origin="local")
876 db, cache, self.APP_INFO_JSON, origin="local")
877 db.close()877 db.close()
878 self.assertTrue(res)878
879 def setUp(self):
880 # create a fake database to simualte a run of software-center-agent
879 # create a StoreDatabase and add our other db881 # create a StoreDatabase and add our other db
880 db = get_test_db()882 self.db = get_test_db()
881 db.add_database(xapian.Database(TEST_DB))883 self.db.add_database(xapian.Database(TEST_DB))
882 db.open(use_axi=True)884 self.db.open(use_axi=True)
883 enquire = AppEnquire(db._aptcache, db)885 self.enquire = AppEnquire(self.db._aptcache, self.db)
886
887 def test_search_app_pkgname_duplication_lp891613(self):
884 # simulate a pkg "apt" that is both in the agent and in the x-a-i db888 # simulate a pkg "apt" that is both in the agent and in the x-a-i db
885 search_term = "apt"889 search_term = "apt"
886 search_query = db.get_query_list_from_search_entry(search_term)890 search_query = self.db.get_query_list_from_search_entry(search_term)
887 enquire.set_query(search_query, nonblocking_load=False)891 self.enquire.set_query(search_query, nonblocking_load=False)
888 self.assertTrue(len(enquire._matches) > 2)892 self.assertTrue(len(self.enquire._matches) > 2)
889 for m in enquire._matches:893 for m in self.enquire._matches:
890 doc = m.document894 doc = m.document
891 # ensure that all hits are "apps" and do not come from a-x-i895 # ensure that all hits are "apps" and do not come from a-x-i
892 self.assertNotEqual(896 self.assertNotEqual(
893 doc.get_value(XapianValues.PKGNAME), "")897 doc.get_value(XapianValues.PKGNAME), "")
894898
899 def test_search_custom_pkgs_list_lp1043159(self):
900 # simulate a pkg "apt" that is both in the agent and in the x-a-i db
901 pkgs = ["apt","gedit"]
902 search_query = self.db.get_query_for_pkgnames(pkgs)
903 self.enquire.set_query(search_query,
904 # custom package lists are always in this mode
905 nonapps_visible=NonAppVisibility.ALWAYS_VISIBLE,
906 nonblocking_load=False)
907 self.assertEqual(len(self.enquire._matches), 2)
908
895909
896if __name__ == "__main__":910if __name__ == "__main__":
897 #import logging911 #import logging

Subscribers

People subscribed via source and target branches