Pastebin: update-mime-database.cのfdatasync呼び出しによる処理時間の問題の対処(for linux)

Format

Diff

Post date

2020-03-11 05:43

Publication Period

Unlimited

 --- a/update-mime-database.c
 +++ b/update-mime-database.c
 @@ -1,3 +1,5 @@
 +#define _GNU_SOURCE
 +
  #include <config.h>
  
  #define N_(x) x
 @@ -37,6 +39,15 @@
  #define NOGLOBS "__NOGLOBS__"
  #define NOMAGIC "__NOMAGIC__"
  
 +#ifdef __linux__
 +# include <linux/version.h>
 +# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 0)
 +#  include <linux/aio_abi.h>
 +#  include <sys/syscall.h>
 +#  include <sys/eventfd.h>
 +# endif
 +#endif
 +
  #ifndef PATH_SEPARATOR
  # ifdef _WIN32
  #  define PATH_SEPARATOR ";"
 @@ -168,12 +179,31 @@
  /* Lists enabled log levels */
  static GLogLevelFlags enabled_log_levels = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL | G_LOG_LEVEL_WARNING;
  
 +static GList *rename_list = NULL;
 +
  /* Static prototypes */
  static Magic *magic_new(xmlNode *node, Type *type, GError **error);
  static Match *match_new(void);
  
  static TreeMagic *tree_magic_new(xmlNode *node, Type *type, GError **error);
  
 +#ifdef __LINUX__AIO_ABI_H
 +static int io_setup(unsigned nr_events, aio_context_t *ctx_idp){
 +       return syscall(SYS_io_setup, nr_events, ctx_idp);
 +}
 +static int io_submit(aio_context_t ctx_id, long nr, struct iocb **iocbpp){
 +       return syscall(SYS_io_submit, ctx_id, nr, iocbpp);
 +}
 +static int io_getevents(aio_context_t ctx_id, long min_nr, long nr,
 +       struct io_event *events, struct timespec *timeout
 +){
 +       return syscall(SYS_io_getevents, ctx_id, min_nr, nr, events, timeout);
 +}
 +static int io_destroy(aio_context_t ctx_id){
 +       return syscall(SYS_io_destroy, ctx_id);
 +}
 +#endif
 +
  static void g_log_handler (const gchar   *log_domain,
                            GLogLevelFlags log_level,
                            const gchar   *message,
 @@ -986,38 +1016,181 @@
         return 0;
  }
  
 -/* Renames pathname by removing the .new extension */
 -static gboolean atomic_update(const gchar *pathname, GError **error)
 +static gboolean add_atomic_update(const gchar *pathname, GError **error)
  {
 -       gboolean ret = FALSE;
 -       gchar *new_name = NULL;
         int len;
  
         len = strlen(pathname);
 -
 +       g_return_val_if_fail(len > 4, FALSE);
         g_return_val_if_fail(strcmp(pathname + len - 4, ".new") == 0, FALSE);
  
 -       new_name = g_strndup(pathname, len - 4);
 +       rename_list = g_list_prepend(rename_list, g_strdup(pathname));
  
 -       if (sync_file(pathname, error) == -1)
 -               goto out;
 +       return TRUE;
 +}
 +
 +static gboolean finish_atomic_update(GError **error){
 +       gboolean ret = FALSE;
 +       GList *listp;
 +       gchar *new_name = NULL;
 +
 +#ifdef __LINUX__AIO_ABI_H
 +       {
 +               guint fsync_cnt = g_list_length(rename_list);
 +               unsigned aio_max_cnt = fsync_cnt;
 +               aio_context_t aio_context = 0;
 +               struct iocb *iocb_array = NULL;
 +               struct io_event *io_event_array = NULL;
 +               guint aio_submited = 0;
 +               unsigned aio_waiting = 0;
 +               int evfd = -1;
 +               int i;
 +
 +               if(!sync_enabled() || fsync_cnt <= 1){
 +                       goto noaio;
 +               }
 +
 +               // try to setup aio
 +               while(io_setup(aio_max_cnt, &aio_context)){
 +                       if(errno == EAGAIN){
 +                               aio_max_cnt = (aio_max_cnt + 1) / 2;
 +                               if(aio_max_cnt <= 1){
 +                                       goto noaio;
 +                               }
 +                       }else{
 +                               goto noaio;
 +                       }
 +               }
 +
 +               if((evfd = eventfd(0, 0)) < 0){
 +                       goto exit_aio;
 +               }
 +
 +               iocb_array = g_malloc0(sizeof(struct iocb) * fsync_cnt);
 +               io_event_array
 +                       = g_malloc(sizeof(struct io_event) * aio_max_cnt);
 +
 +               // do fdatasync using aio abi
 +               for(listp = rename_list; listp; listp = listp->next){
 +                       gchar *pathname = listp->data;
 +                       struct iocb *iocbp;
 +                       int fd;
 +
 +                       while(aio_waiting >= aio_max_cnt){
 +                               int fin_cnt;
 +                               fin_cnt = io_getevents(aio_context,
 +                                       1, aio_max_cnt, io_event_array, NULL);
 +                               if(fin_cnt < 0){
 +                                       if(errno == EINTR){
 +                                               continue;
 +                                       }
 +                                       goto exit_aio;
 +                               }
 +                               for(i = 0; i < fin_cnt; i++){
 +                                       __u32 finfd = iocb_array[
 +                                               io_event_array[i].data
 +                                       ].aio_fildes;
 +                                       close(finfd);
 +                                       iocb_array[io_event_array[i].data]
 +                                               .aio_data = -1;
 +                               }
 +                               aio_waiting -= fin_cnt;
 +                       }
 +
 +                       if((fd = open(pathname, O_RDWR)) < 0){
 +                               continue;
 +                       }
 +
 +                       iocbp = &iocb_array[aio_submited];
 +                       iocbp->aio_data = aio_submited;
 +                       //iocbp->aio_rw_flags = 0;
 +                       iocbp->aio_lio_opcode = IOCB_CMD_FDSYNC;
 +                       //iocbp->aio_reqprio = 0;
 +                       iocbp->aio_fildes = fd;
 +                       //iocbp->aio_buf = 0;
 +                       //iocbp->aio_nbytes = 0;
 +                       //iocbp->aio_offset = 0;
 +                       //iocbp->aio_flags = 0;
 +                       iocbp->aio_resfd = evfd;
 +
 +                       if(io_submit(aio_context, (long)1, &iocbp) != 1){
 +                               close(fd);
 +                               continue;
 +                       }
 +
 +                       aio_waiting++;
 +                       aio_submited++;
 +               }
 +
 +               while(aio_waiting > 0){
 +                       int fin_cnt;
 +                       fin_cnt = io_getevents(aio_context,
 +                               aio_waiting, aio_max_cnt, io_event_array, NULL);
 +                       if(fin_cnt < 0){
 +                               if(errno == EINTR){
 +                                       continue;
 +                               }
 +                               goto exit_aio;
 +                       }
 +                       aio_waiting -= fin_cnt;
 +               }
 +
 +exit_aio:
 +               io_destroy(aio_context);
 +               if(evfd >= 0){
 +                       close(evfd);
 +               }
 +               for(i = 0; i < aio_submited; i++){
 +                       struct iocb *iocbp = &iocb_array[i];
 +                       if(iocbp->aio_data != (__u64)-1){
 +                               close(iocbp->aio_fildes);
 +                       }
 +               }
 +
 +noaio:
 +               g_free(iocb_array);
 +               g_free(io_event_array);
 +       }
 +#endif
 +
 +       for(listp = g_list_last(rename_list); listp; listp = listp->prev){
 +               int len;
 +               gchar *pathname = listp->data;
 +
 +               len = strlen(pathname);
 +               g_return_val_if_fail(len > 4, FALSE);
 +               g_return_val_if_fail(
 +                       strcmp(pathname + len - 4, ".new") == 0, FALSE);
 +
 +               new_name = g_strndup(pathname, len - 4);
 +
 +               if (sync_file(pathname, error) == -1)
 +                       goto out;
  
  #ifdef _WIN32
 -       /* we need to remove the old file first! */
 -       remove(new_name);
 +               /* we need to remove the old file first! */
 +               remove(new_name);
  #endif
 -       if (rename(pathname, new_name) == -1)
 -       {
 -               int errsv = errno;
 -               g_set_error(error, G_FILE_ERROR, g_file_error_from_errno(errsv),
 -                           "Failed to rename %s as %s: %s", pathname, new_name,
 -                           g_strerror(errsv));
 -               goto out;
 +
 +               if (rename(pathname, new_name) == -1)
 +               {
 +                       int errsv = errno;
 +                       g_set_error(error, G_FILE_ERROR,
 +                               g_file_error_from_errno(errsv),
 +                               "Failed to rename %s as %s: %s",
 +                               pathname, new_name, g_strerror(errsv));
 +                       goto out;
 +               }
 +
 +               g_free(new_name);
 +               new_name = NULL;
         }
  
         ret = TRUE;
  out:
         g_free(new_name);
 +       g_list_free_full(rename_list, g_free);
 +       rename_list = NULL;
         return ret;
  }
  
 @@ -1048,7 +1221,7 @@
         if (!save_xml_file(type->output, filename, &local_error))
                 fatal_gerror(local_error);
  
 -       if (!atomic_update(filename, &local_error))
 +       if (!add_atomic_update(filename, &local_error))
                 fatal_gerror(local_error);
  
         g_free(filename);
 @@ -3756,7 +3929,7 @@
                 write_out_glob(glob_list, globs);
                 if (!fclose_gerror(globs, error))
                         goto out;
 -               if (!atomic_update(globs_path, error))
 +               if (!add_atomic_update(globs_path, error))
                         goto out;
                 g_free(globs_path);
  
 @@ -3770,7 +3943,7 @@
                 write_out_glob2 (glob_list, globs);
                 if (!fclose_gerror(globs, error))
                         goto out;
 -               if (!atomic_update(globs_path, error))
 +               if (!add_atomic_update(globs_path, error))
                         goto out;
                 g_free(globs_path);
  
 @@ -3797,7 +3970,7 @@
                 }
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(magic_path, error))
 +               if (!add_atomic_update(magic_path, error))
                         goto out;
                 g_free(magic_path);
         }
 @@ -3813,7 +3986,7 @@
                 write_namespaces(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(ns_path, error))
 +               if (!add_atomic_update(ns_path, error))
                         goto out;
                 g_free(ns_path);
         }
 @@ -3829,7 +4002,7 @@
                 write_subclasses(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3845,7 +4018,7 @@
                 write_aliases(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3861,7 +4034,7 @@
                 write_types(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3877,7 +4050,7 @@
                 write_icons(generic_icon_hash, stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(icon_path, error))
 +               if (!add_atomic_update(icon_path, error))
                         goto out;
                 g_free(icon_path);
         }
 @@ -3893,7 +4066,7 @@
                 write_icons(icon_hash, stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(icon_path, error))
 +               if (!add_atomic_update(icon_path, error))
                         goto out;
                 g_free(icon_path);
         }
 @@ -3918,7 +4091,7 @@
                 }
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3934,7 +4107,7 @@
                 write_cache(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3951,11 +4124,15 @@
                           VERSION "\n");
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
  
 +       if (!finish_atomic_update(error)){
 +               goto out;
 +       }
 +
         g_ptr_array_foreach(magic_array, (GFunc)magic_free, NULL);
         g_ptr_array_free(magic_array, TRUE);
         g_ptr_array_foreach(tree_magic_array, (GFunc)tree_magic_free, NULL);