GNOME Bugzilla – Bug 511474
The option for Don't index remote mounted directories doesn't work in some cases
Last modified: 2010-06-25 10:21:09 UTC
Please describe the problem: The mechanism we use to check remote mounted doesn't work in some cases. A patch for this: Index: src/trackerd/tracker-utils.h =================================================================== --- src/trackerd/tracker-utils.h (revision 1055) +++ src/trackerd/tracker-utils.h (working copy) @@ -308,7 +308,7 @@ const char *current_uri; gboolean skip_mount_points; /* should tracker descend into mounted directories? see Tracker.root_directory_devices */ - GSList * root_directory_devices; + GHashTable *remote_directories; IndexStatus index_status; @@ -591,9 +591,7 @@ gboolean tracker_file_is_no_watched (const char *uri); gboolean tracker_file_is_crawled (const char *uri); -void tracker_add_root_dir (const char *uri); /* add a directory to the list of watch/crawl/service roots */ -void tracker_add_root_directories (GSList *uri_list); /* adds a bunch of directories to the list of watch/crawl/service roots */ -gboolean tracker_file_is_in_root_dir (const char *uri); /* test if a given file resides in the watch/crawl/service roots */ +gboolean tracker_file_is_remote_mounted (const char *uri); GSList * tracker_get_all_files (const char *dir, gboolean dir_only); GSList * tracker_get_files (const char *dir, gboolean dir_only); Index: src/trackerd/trackerd.c =================================================================== --- src/trackerd/trackerd.c (revision 1057) +++ src/trackerd/trackerd.c (working copy) @@ -1054,8 +1054,6 @@ list = tracker_get_service_dirs ("Applications"); - tracker_add_root_directories (list); - process_directory_list (db_con, list, FALSE); tracker_db_end_transaction (db_con->cache); @@ -1120,8 +1118,6 @@ tracker_db_start_transaction (db_con->cache); - tracker_add_root_directories (tracker->watch_directory_roots_list); - /* index watched dirs first */ g_slist_foreach (tracker->watch_directory_roots_list, (GFunc) watch_dir, db_con); @@ -1158,8 +1154,6 @@ tracker_db_start_transaction (db_con->cache); - tracker_add_root_directories (tracker->crawl_directory_list); - add_dirs_to_list (tracker->crawl_directory_list, db_con); g_slist_foreach (tracker->dir_list, (GFunc) schedule_dir_check, db_con); @@ -1212,7 +1206,6 @@ if (has_logs) { tracker_log ("Starting chat log indexing..."); tracker_db_start_transaction (db_con->cache); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); tracker_db_end_transaction (db_con->cache); g_slist_free (list); @@ -1256,7 +1249,6 @@ if (tracker->index_evolution_emails) { GSList *list = tracker_get_service_dirs ("EvolutionEmails"); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); g_slist_free (list); @@ -1275,14 +1267,12 @@ if (tracker->index_kmail_emails) { GSList *list = tracker_get_service_dirs ("KMailEmails"); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); g_slist_free (list); } if (tracker->index_thunderbird_emails) { GSList *list = tracker_get_service_dirs ("ThunderbirdEmails"); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); g_slist_free (list); } @@ -2128,7 +2118,7 @@ tracker->services_dir = g_build_filename (TRACKER_DATADIR, "tracker", "services", NULL); tracker->skip_mount_points = FALSE; - tracker->root_directory_devices = NULL; + tracker->remote_directories = NULL; tracker->folders_count = 0; tracker->folders_processed = 0; Index: src/trackerd/tracker-utils.c =================================================================== --- src/trackerd/tracker-utils.c (revision 1055) +++ src/trackerd/tracker-utils.c (working copy) @@ -40,6 +40,15 @@ #include <glib/gpattern.h> #include <zlib.h> #include <math.h> + +#ifdef HAVE_GETMNTENT +#if defined(__linux__) +#include <mntent.h> +#elif defined(__sun) +#include <sys/mnttab.h> +#endif +#endif + #include "tracker-dbus.h" #include "tracker-utils.h" #include "tracker-indexer.h" @@ -105,6 +114,12 @@ {NULL, NULL}, }; +static const char * g_remote_fs_type_list[] = {"nfs", "nfs3", "nfs4", "smbfs", "sshfs", "ftpfs", NULL}; // autofs, ctfs ? +#ifdef __linux__ +#define g_mnt_tab MOUNTED +#elif defined(__sun) +#define g_mnt_tab MNTTAB +#endif char * tracker_get_service_by_id (int service_type_id) @@ -1176,92 +1191,84 @@ } -void -tracker_add_root_dir (const char *uri) +#ifdef HAVE_GETMNTENT +static gboolean +tracker_is_remote_fs_type(const char *fs_type) { - struct stat st; - - if (! tracker->skip_mount_points) { - return; + int i; + for (i = 0; NULL != g_remote_fs_type_list[i]; i++) { + if (g_str_equal(g_remote_fs_type_list[i], fs_type)) + return TRUE; } - if (!uri || uri[0] != '/') { + return FALSE; +} + + +static void +tracker_init_remote_dirs() +{ + if (tracker->remote_directories) return; - } - if (g_stat (uri, &st) == 0) { - GSList * cur = NULL; - dev_t * elem = NULL; + tracker->remote_directories = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); - if (! S_ISDIR (st.st_mode)) { - return; - } +#ifdef __linux__ + FILE *file = setmntent(g_mnt_tab, "r"); + if (!file) + return; - /* FIXME: too costly? */ - for (cur = tracker->root_directory_devices; cur; cur = g_slist_next (cur)) { - if (cur->data && *((dev_t *) cur->data) == st.st_dev) { - return; - } - } + struct mntent * ent = getmntent(file); + while (ent) { + if (tracker_is_remote_fs_type(ent->mnt_type)) + g_hash_table_insert (tracker->remote_directories, g_strdup(ent->mnt_dir), NULL); - elem = g_new (dev_t, 1); - * elem = st.st_dev; - tracker->root_directory_devices = g_slist_prepend (tracker->root_directory_devices, elem); - } else { - tracker_log ("Could not stat `%s'", uri); + ent = getmntent(file); } -} + endmntent(file); +#elif defined(__sun) + FILE *file = fopen(g_mnt_tab, "r"); + if (!file) + return; -void -tracker_add_root_directories (GSList * uri_list) -{ - GSList * cur = NULL; + struct mnttab ent; + int rtn = getmntent(file, &ent); + while (rtn == 0) + { + if (tracker_is_remote_fs_type(ent.mnt_fstype)) + g_hash_table_insert (tracker->remote_directories, g_strdup(ent.mnt_mountp), NULL); - if (! tracker->skip_mount_points) { - return; + rtn = getmntent(file, &ent); } +#else +#error tracker_init_remote_dirs not implemented for this platform +#endif - for (cur = uri_list; cur; cur = g_slist_next (cur)) { - tracker_add_root_dir ((const char *) cur->data); - } + return; } gboolean -tracker_file_is_in_root_dir (const char *uri) +tracker_file_is_remote_mounted (const char *uri) { - struct stat st; - GSList * cur = NULL; - dev_t uri_dev = 0; - - if (! tracker->skip_mount_points) { - return TRUE; + if (!tracker->remote_directories) { + tracker_init_remote_dirs(); } - if (!uri || uri[0] != '/') { - return FALSE; + if (tracker->remote_directories) { + return g_hash_table_lookup_extended(tracker->remote_directories, uri, NULL, NULL); } - if (g_stat (uri, &st) == 0) { - uri_dev = st.st_dev; - } else { - tracker_log ("Could not stat `%s'", uri); - return TRUE; /* the caller should take care of skipping this one */ - } - - if (! S_ISDIR (st.st_mode)) { /* only directories are mount points and therefore checked */ - return TRUE; - } - - for (cur = tracker->root_directory_devices; cur; cur = g_slist_next (cur)) { - if (cur->data && *((dev_t *) cur->data) == uri_dev) { - return TRUE; - } - } - return FALSE; } +#else +gboolean +tracker_file_is_remote_mounted (const char *uri) +{ + return FALSE; +} +#endif gboolean @@ -1993,7 +2000,7 @@ - if (!tracker_file_is_in_root_dir (mystr)) { + if (tracker->skip_mount_points && tracker_file_is_remote_mounted (mystr)) { tracker_log ("Skipping mount point %s", mystr); g_free (mystr); continue; Index: configure.ac =================================================================== --- configure.ac (revision 1055) +++ configure.ac (working copy) @@ -738,6 +738,11 @@ test "$have_libgsf" = "yes" && AC_DEFINE(HAVE_LIBGSF, [], [Define if we have libgsf]) #################################################################### +# Check getmntent +#################################################################### +AC_FUNC_GETMNTENT + +#################################################################### # Check ioprio support #################################################################### Subject: Re: pls help to review this patch for remote mount checking From: Halton Huo <Halton.Huo@Sun.COM> Date: Thu, 06 Dec 2007 14:49:16 +0800 To: Rick Ju <Rick.Ju@Sun.COM> CC: "Jerry.Tan" <Jerry.Tan@Sun.COM> Works fine on Solaris. But not work folder which is for a soft link to remote, eg: ln -s /net/firstserve.prc/export/so /export/home/halton/test/firstserve On Thu, 2007-12-06 at 14:14 +0800, Rick Ju wrote: > > Versio Index: src/trackerd/tracker-utils.h =================================================================== --- src/trackerd/tracker-utils.h (revision 1055) +++ src/trackerd/tracker-utils.h (working copy) @@ -308,7 +308,7 @@ const char *current_uri; gboolean skip_mount_points; /* should tracker descend into mounted directories? see Tracker.root_directory_devices */ - GSList * root_directory_devices; + GHashTable *remote_directories; IndexStatus index_status; @@ -591,9 +591,7 @@ gboolean tracker_file_is_no_watched (const char *uri); gboolean tracker_file_is_crawled (const char *uri); -void tracker_add_root_dir (const char *uri); /* add a directory to the list of watch/crawl/service roots */ -void tracker_add_root_directories (GSList *uri_list); /* adds a bunch of directories to the list of watch/crawl/service roots */ -gboolean tracker_file_is_in_root_dir (const char *uri); /* test if a given file resides in the watch/crawl/service roots */ +gboolean tracker_file_is_remote_mounted (const char *uri); GSList * tracker_get_all_files (const char *dir, gboolean dir_only); GSList * tracker_get_files (const char *dir, gboolean dir_only); Index: src/trackerd/trackerd.c =================================================================== --- src/trackerd/trackerd.c (revision 1057) +++ src/trackerd/trackerd.c (working copy) @@ -1054,8 +1054,6 @@ list = tracker_get_service_dirs ("Applications"); - tracker_add_root_directories (list); - process_directory_list (db_con, list, FALSE); tracker_db_end_transaction (db_con->cache); @@ -1120,8 +1118,6 @@ tracker_db_start_transaction (db_con->cache); - tracker_add_root_directories (tracker->watch_directory_roots_list); - /* index watched dirs first */ g_slist_foreach (tracker->watch_directory_roots_list, (GFunc) watch_dir, db_con); @@ -1158,8 +1154,6 @@ tracker_db_start_transaction (db_con->cache); - tracker_add_root_directories (tracker->crawl_directory_list); - add_dirs_to_list (tracker->crawl_directory_list, db_con); g_slist_foreach (tracker->dir_list, (GFunc) schedule_dir_check, db_con); @@ -1212,7 +1206,6 @@ if (has_logs) { tracker_log ("Starting chat log indexing..."); tracker_db_start_transaction (db_con->cache); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); tracker_db_end_transaction (db_con->cache); g_slist_free (list); @@ -1256,7 +1249,6 @@ if (tracker->index_evolution_emails) { GSList *list = tracker_get_service_dirs ("EvolutionEmails"); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); g_slist_free (list); @@ -1275,14 +1267,12 @@ if (tracker->index_kmail_emails) { GSList *list = tracker_get_service_dirs ("KMailEmails"); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); g_slist_free (list); } if (tracker->index_thunderbird_emails) { GSList *list = tracker_get_service_dirs ("ThunderbirdEmails"); - tracker_add_root_directories (list); process_directory_list (db_con, list, TRUE); g_slist_free (list); } @@ -2128,7 +2118,7 @@ tracker->services_dir = g_build_filename (TRACKER_DATADIR, "tracker", "services", NULL); tracker->skip_mount_points = FALSE; - tracker->root_directory_devices = NULL; + tracker->remote_directories = NULL; tracker->folders_count = 0; tracker->folders_processed = 0; Index: src/trackerd/tracker-utils.c =================================================================== --- src/trackerd/tracker-utils.c (revision 1055) +++ src/trackerd/tracker-utils.c (working copy) @@ -40,6 +40,15 @@ #include <glib/gpattern.h> #include <zlib.h> #include <math.h> + +#ifdef HAVE_GETMNTENT +#if defined(__linux__) +#include <mntent.h> +#elif defined(__sun) +#include <sys/mnttab.h> +#endif +#endif + #include "tracker-dbus.h" #include "tracker-utils.h" #include "tracker-indexer.h" @@ -105,6 +114,12 @@ {NULL, NULL}, }; +static const char * g_remote_fs_type_list[] = {"nfs", "nfs3", "nfs4", "smbfs", "sshfs", "ftpfs", NULL}; // autofs, ctfs ? +#ifdef __linux__ +#define g_mnt_tab MOUNTED +#elif defined(__sun) +#define g_mnt_tab MNTTAB +#endif char * tracker_get_service_by_id (int service_type_id) @@ -1176,92 +1191,84 @@ } -void -tracker_add_root_dir (const char *uri) +#ifdef HAVE_GETMNTENT +static gboolean +tracker_is_remote_fs_type(const char *fs_type) { - struct stat st; - - if (! tracker->skip_mount_points) { - return; + int i; + for (i = 0; NULL != g_remote_fs_type_list[i]; i++) { + if (g_str_equal(g_remote_fs_type_list[i], fs_type)) + return TRUE; } - if (!uri || uri[0] != '/') { + return FALSE; +} + + +static void +tracker_init_remote_dirs() +{ + if (tracker->remote_directories) return; - } - if (g_stat (uri, &st) == 0) { - GSList * cur = NULL; - dev_t * elem = NULL; + tracker->remote_directories = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); - if (! S_ISDIR (st.st_mode)) { - return; - } +#ifdef __linux__ + FILE *file = setmntent(g_mnt_tab, "r"); + if (!file) + return; - /* FIXME: too costly? */ - for (cur = tracker->root_directory_devices; cur; cur = g_slist_next (cur)) { - if (cur->data && *((dev_t *) cur->data) == st.st_dev) { - return; - } - } + struct mntent * ent = getmntent(file); + while (ent) { + if (tracker_is_remote_fs_type(ent->mnt_type)) + g_hash_table_insert (tracker->remote_directories, g_strdup(ent->mnt_dir), NULL); - elem = g_new (dev_t, 1); - * elem = st.st_dev; - tracker->root_directory_devices = g_slist_prepend (tracker->root_directory_devices, elem); - } else { - tracker_log ("Could not stat `%s'", uri); + ent = getmntent(file); } -} + endmntent(file); +#elif defined(__sun) + FILE *file = fopen(g_mnt_tab, "r"); + if (!file) + return; -void -tracker_add_root_directories (GSList * uri_list) -{ - GSList * cur = NULL; + struct mnttab ent; + int rtn = getmntent(file, &ent); + while (rtn == 0) + { + if (tracker_is_remote_fs_type(ent.mnt_fstype)) + g_hash_table_insert (tracker->remote_directories, g_strdup(ent.mnt_mountp), NULL); - if (! tracker->skip_mount_points) { - return; + rtn = getmntent(file, &ent); } +#else +#error tracker_init_remote_dirs not implemented for this platform +#endif - for (cur = uri_list; cur; cur = g_slist_next (cur)) { - tracker_add_root_dir ((const char *) cur->data); - } + return; } gboolean -tracker_file_is_in_root_dir (const char *uri) +tracker_file_is_remote_mounted (const char *uri) { - struct stat st; - GSList * cur = NULL; - dev_t uri_dev = 0; - - if (! tracker->skip_mount_points) { - return TRUE; + if (!tracker->remote_directories) { + tracker_init_remote_dirs(); } - if (!uri || uri[0] != '/') { - return FALSE; + if (tracker->remote_directories) { + return g_hash_table_lookup_extended(tracker->remote_directories, uri, NULL, NULL); } - if (g_stat (uri, &st) == 0) { - uri_dev = st.st_dev; - } else { - tracker_log ("Could not stat `%s'", uri); - return TRUE; /* the caller should take care of skipping this one */ - } - - if (! S_ISDIR (st.st_mode)) { /* only directories are mount points and therefore checked */ - return TRUE; - } - - for (cur = tracker->root_directory_devices; cur; cur = g_slist_next (cur)) { - if (cur->data && *((dev_t *) cur->data) == uri_dev) { - return TRUE; - } - } - return FALSE; } +#else +gboolean +tracker_file_is_remote_mounted (const char *uri) +{ + return FALSE; +} +#endif gboolean @@ -1993,7 +2000,7 @@ - if (!tracker_file_is_in_root_dir (mystr)) { + if (tracker->skip_mount_points && tracker_file_is_remote_mounted (mystr)) { tracker_log ("Skipping mount point %s", mystr); g_free (mystr); continue; Index: configure.ac =================================================================== --- configure.ac (revision 1055) +++ configure.ac (working copy) @@ -738,6 +738,11 @@ test "$have_libgsf" = "yes" && AC_DEFINE(HAVE_LIBGSF, [], [Define if we have libgsf]) #################################################################### +# Check getmntent +#################################################################### +AC_FUNC_GETMNTENT + +#################################################################### # Check ioprio support #################################################################### Subject: Re: pls help to review this patch for remote mount checking From: Halton Huo <Halton.Huo@Sun.COM> Date: Thu, 06 Dec 2007 14:49:16 +0800 To: Rick Ju <Rick.Ju@Sun.COM> CC: "Jerry.Tan" <Jerry.Tan@Sun.COM> Works fine on Solaris. But not work folder which is for a soft link to remote, eg: ln -s /net/firstserve.prc/export/so /export/home/halton/test/firstserve On Thu, 2007-12-06 at 14:14 +0800, Rick Ju wrote: > > Version 2 > > > > Rick Ju wrote: >> > > n 2 > > > > Rick Ju wrote: >> > > Steps to reproduce: 1. 2. 3. Actual results: Expected results: Does this happen every time? Other information:
Is anyone looking at this? I'm having a slightly different problem (I think), which is that I have automount entries registered and trackerd (I think it's trackerd) is constantly trying to open <mntpoint>/.Trash where <mntpoint> is each automount mount point. Not only does this fill the logs with useless error messages but it slows down the system especially for very large maps.
Still hoping someone is looking at this. It's extremely annoying to say the least and as I mentioned, if you have a lot of automount points it can cause your system load to skyrocket when tracker tries to look for things. Lei, can you please add the patch you've given as an attachment to the bug (see the end of the bug page) instead of pasting it inline? Pasting it destroys all the formatting and there's no way to apply a patch like that. If you attach it then it can be saved and applied. Thanks.
I guess no one cares about this software; that being the case I'm disabling it on my system. Automount is more important to me than tracker, and right now this software is causing my system to hang every single night, such that I have to reboot it in the morning. Information in the logs etc. seems to point to this broken indexing of remote directories as the instigator of the problem. If anyone cares, there's more info here: https://bugs.launchpad.net/ubuntu/+source/autofs/+bug/190737 Also if anyone cares, I'm willing to do some work to try to help track this down. Let me know what info you need and I'll re-enable tracker to try to help find it.
Can the reporter of this bug set the _version_ so we can target bugs to work on more easily please. We can also have an idea about if these bugs are likely to be obsolete too. If you don't set the version, we are less likely to look at them. Sorry for the spam, but I don't want to say this 88 times on each bug :)
I noticed this too is broken on master, it should be fixed or the option removed from the config. (In reply to comment #1) > Is anyone looking at this? I'm having a slightly different problem (I think), > which is that I have automount entries registered and trackerd (I think it's > trackerd) is constantly trying to open <mntpoint>/.Trash where <mntpoint> is > each automount mount point. It is on the list of things to do.
Created attachment 164598 [details] [review] Remove the config option
Fix commited to git master. In a brief, if you ever want to index a given remote mount, just add the path where it's mounted to the list of recursively indexed directories. This problem has been fixed in the development version. The fix will be available in the next major software release. Thank you for your bug report.