diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2011-08-10 22:00:34 +0000 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2011-08-10 22:00:34 +0000 |
commit | 74f7fe561e31637bdbf21aeb817d4a22bbf9c91a (patch) | |
tree | bae9da9f2c749d290ad4f1764dc205bcd23b750e | |
parent | 6b6dcc80555d77858733e44b72e92d1dc60e2c38 (diff) | |
download | gdbm-74f7fe561e31637bdbf21aeb817d4a22bbf9c91a.tar.gz gdbm-74f7fe561e31637bdbf21aeb817d4a22bbf9c91a.tar.bz2 |
Improve memory mapping support.
The new code is more flexible and performs better when
lots of inserts are being made (e.g. when populating the
database with new data).
* src/gdbm.h.in (GDBM_SETMAXMAPSIZE): New constant.
* src/gdbmconst.h (SIZE_T_MAX): New define.
* src/gdbmdefs.h (gdbm_file_info) <cache_size>: Change type
to size_t.
<mmap_inited,mapped_size_max>: New member.
<mapped_remap>: Remove.
* src/gdbmopen.c: Fix a typo.
(gdbm_open): Initialize new members.
(_gdbm_init_cache): Second argument is size_t.
* src/gdbmsetopt.c (gdbm_setopt): Optval argument is void*.
Handle GDBM_SETMAXMAPSIZE.
Improve error checking.
* src/mmap.c (_GDBM_IN_MAPPED_REGION_P): Fix comparison with
the lower bound.
(_GDBM_NEED_REMAP): Return true if mapped_region is NULL.
(SUM_FILE_SIZE): Rewrite.
(_gdbm_mapped_unmap): Don't call msync.
(_gdbm_internal_remap): Take 2 arguments, the second one
giving the new mapped size.
Unmap the region prior to remapping it.
Always pass NULL as the argument to mmap.
(_gdbm_mapped_remap): Rewrite the logic. Change semantics of the
third argument. All uses updated.
(_gdbm_mapped_init): Reflect the above changes.
(_gdbm_mapped_read,_gdbm_mapped_write): Use mmap_inited to decide
whether to use mmap, because mapped_region can be reset to zero
by another functions (namely, _gdbm_mapped_lseek).
Reset mmap_inited to FALSE, if _gdbm_mapped_remap fails.
(_gdbm_mapped_lseek): Rewrite offset computations. Invalidate
the mapped region.
* src/proto.h (_gdbm_init_cache): Change prototype.
* src/update.c (write_header, _gdbm_end_update): Remove checks
for dbf->mapped_region.
* tests/gtload.c: Implement the -maxmap option (set maximal
mapped memory size).
* doc/gdbm.texinfo: Document GDBM_SETMAXMAPSIZE.
-rw-r--r-- | ChangeLog | 46 | ||||
-rw-r--r-- | doc/gdbm.texinfo | 10 | ||||
-rw-r--r-- | src/gdbm.h.in | 4 | ||||
-rw-r--r-- | src/gdbmconst.h | 3 | ||||
-rw-r--r-- | src/gdbmdefs.h | 12 | ||||
-rw-r--r-- | src/gdbmopen.c | 12 | ||||
-rw-r--r-- | src/gdbmsetopt.c | 98 | ||||
-rw-r--r-- | src/mmap.c | 145 | ||||
-rw-r--r-- | src/proto.h | 2 | ||||
-rw-r--r-- | src/update.c | 7 | ||||
-rw-r--r-- | tests/gtload.c | 35 |
11 files changed, 259 insertions, 115 deletions
@@ -1,3 +1,49 @@ +2011-08-11 Sergey Poznyakoff <gray@gnu.org.ua> + + Improve memory mapping support. + + The new code is more flexible and performs better when + lots of inserts are being made (e.g. when populating the + database with new data). + + * src/gdbm.h.in (GDBM_SETMAXMAPSIZE): New constant. + * src/gdbmconst.h (SIZE_T_MAX): New define. + * src/gdbmdefs.h (gdbm_file_info) <cache_size>: Change type + to size_t. + <mmap_inited,mapped_size_max>: New member. + <mapped_remap>: Remove. + * src/gdbmopen.c: Fix a typo. + (gdbm_open): Initialize new members. + (_gdbm_init_cache): Second argument is size_t. + * src/gdbmsetopt.c (gdbm_setopt): Optval argument is void*. + Handle GDBM_SETMAXMAPSIZE. + Improve error checking. + * src/mmap.c (_GDBM_IN_MAPPED_REGION_P): Fix comparison with + the lower bound. + (_GDBM_NEED_REMAP): Return true if mapped_region is NULL. + (SUM_FILE_SIZE): Rewrite. + (_gdbm_mapped_unmap): Don't call msync. + (_gdbm_internal_remap): Take 2 arguments, the second one + giving the new mapped size. + Unmap the region prior to remapping it. + Always pass NULL as the argument to mmap. + (_gdbm_mapped_remap): Rewrite the logic. Change semantics of the + third argument. All uses updated. + (_gdbm_mapped_init): Reflect the above changes. + (_gdbm_mapped_read,_gdbm_mapped_write): Use mmap_inited to decide + whether to use mmap, because mapped_region can be reset to zero + by another functions (namely, _gdbm_mapped_lseek). + Reset mmap_inited to FALSE, if _gdbm_mapped_remap fails. + (_gdbm_mapped_lseek): Rewrite offset computations. Invalidate + the mapped region. + * src/proto.h (_gdbm_init_cache): Change prototype. + * src/update.c (write_header, _gdbm_end_update): Remove checks + for dbf->mapped_region. + * tests/gtload.c: Implement the -maxmap option (set maximal + mapped memory size). + + * doc/gdbm.texinfo: Document GDBM_SETMAXMAPSIZE. + 2011-08-09 Sergey Poznyakoff <gray@gnu.org.ua> Update the docs. diff --git a/doc/gdbm.texinfo b/doc/gdbm.texinfo index c349941..5dc8a97 100644 --- a/doc/gdbm.texinfo +++ b/doc/gdbm.texinfo @@ -738,7 +738,7 @@ The @var{errno} argument is usually the value of the global variable open database. @deftypefn {gdbm interface} int gdbm_setopt (GDBM_FILE @var{dbf}, int @var{option}, @ - int *@var{value}, int @var{size}) + void *@var{value}, int @var{size}) Sets an option on the database. The parameters are: @@ -809,6 +809,14 @@ a @acronym{CPU} expensive process with time, though, especially if used in conjunction with GDBM_CENTFREE. The @var{value} should point to an integer: @samp{TRUE} to turn free block merging on, and @samp{FALSE} to turn it off. + +@kwindex GDBM_SETMAXMAPSIZE +@item GDBM_SETMAXMAPSIZE +Sets maximum size of the memory mapped region. The @var{value} should +point to a value of type @code{size_t}, @code{unsigned long} or +@code{unsigned}. The actual value is rounded to the nearest page +boundary (the page size is obtained from +@code{sysconf(_SC_PAGESIZE)}). @end table The return value will be @samp{-1} upon failure, or @samp{0} upon diff --git a/src/gdbm.h.in b/src/gdbm.h.in index 70de4e0..8eaee79 100644 --- a/src/gdbm.h.in +++ b/src/gdbm.h.in @@ -51,6 +51,7 @@ #define GDBM_SYNCMODE 3 /* Turn on or off sync operations. */ #define GDBM_CENTFREE 4 /* Keep all free blocks in the header. */ #define GDBM_COALESCEBLKS 5 /* Attempt to coalesce free blocks. */ +#define GDBM_SETMAXMAPSIZE 6 /* Set maximum mapped memory size */ /* The data and key structure. */ typedef struct { @@ -89,7 +90,7 @@ extern datum gdbm_nextkey (GDBM_FILE, datum); extern int gdbm_reorganize (GDBM_FILE); extern void gdbm_sync (GDBM_FILE); extern int gdbm_exists (GDBM_FILE, datum); -extern int gdbm_setopt (GDBM_FILE, int, int *, int); +extern int gdbm_setopt (GDBM_FILE, int, void *, int); extern int gdbm_fdesc (GDBM_FILE); extern int gdbm_export (GDBM_FILE, const char *, int, int); extern int gdbm_import (GDBM_FILE, const char *, int); @@ -136,6 +137,7 @@ extern "C" { #endif extern const char *gdbm_strerror (gdbm_error); +extern int gdbm_version_cmp (int const a[], int const b[]); #if defined(__cplusplus) || defined(c_plusplus) } diff --git a/src/gdbmconst.h b/src/gdbmconst.h index a4c8cfd..7a3f5b9 100644 --- a/src/gdbmconst.h +++ b/src/gdbmconst.h @@ -70,3 +70,6 @@ /* The size of the bucket cache. */ #define DEFAULT_CACHESIZE 100 + +/* Maximum size representable by a size_t variable */ +#define SIZE_T_MAX ((size_t)-1) diff --git a/src/gdbmdefs.h b/src/gdbmdefs.h index 819abbc..2221aa5 100644 --- a/src/gdbmdefs.h +++ b/src/gdbmdefs.h @@ -124,8 +124,6 @@ typedef struct { data_cache_elem ca_data; } cache_elem; - - /* This final structure contains all main memory based information for a gdbm file. This allows multiple gdbm files to be opened at the same time by one program. */ @@ -153,7 +151,8 @@ struct gdbm_file_info { /* Whether or not we're allowing mmap() use. */ unsigned allow_mmap :1; - + unsigned mmap_inited :1; + /* Type of file locking in use. */ enum { LOCKING_NONE = 0, LOCKING_FLOCK, LOCKING_LOCKF, LOCKING_FCNTL } lock_type; @@ -173,7 +172,7 @@ struct gdbm_file_info { /* The bucket cache. */ cache_elem *bucket_cache; - int cache_size; + size_t cache_size; int last_read; /* Points to the current hash bucket in the cache. */ @@ -193,15 +192,12 @@ struct gdbm_file_info { unsigned second_changed :1; /* Mmap info */ + size_t mapped_size_max;/* Max. allowed value for mapped_size */ void *mapped_region; /* Mapped region */ size_t mapped_size; /* Size of the region */ off_t mapped_pos; /* Current offset in the region */ off_t mapped_off; /* Position in the file where the region begins */ - int mapped_remap; /* When set, any call to - _gdbm_mapped_{write|read} will remap the - region according to the above fields. */ - }; /* Now define all the routines in use. */ diff --git a/src/gdbmopen.c b/src/gdbmopen.c index 7cd9040..a110d64 100644 --- a/src/gdbmopen.c +++ b/src/gdbmopen.c @@ -28,7 +28,7 @@ #elif SIZEOF_OFF_T == 8 #define GDBM_MAGIC GDBM_MAGIC64 #elif -#error "Unsupport off_t size, contact GDBM maintainer. What crazy system is this?!?" +#error "Unsupported off_t size, contact GDBM maintainer. What crazy system is this?!?" #endif /* Initialize dbm system. FILE is a pointer to the file name. If the file @@ -84,6 +84,8 @@ gdbm_open (const char *file, int block_size, int flags, int mode, dbf->bucket_cache = NULL; dbf->cache_size = 0; + dbf->mmap_inited = FALSE; + dbf->mapped_size_max = SIZE_T_MAX; dbf->mapped_region = NULL; dbf->mapped_size = 0; dbf->mapped_pos = 0; @@ -108,7 +110,7 @@ gdbm_open (const char *file, int block_size, int flags, int mode, dbf->central_free = FALSE; /* Default to not using central_free. */ dbf->coalesce_blocks = FALSE; /* Default to not coalescing blocks. */ dbf->allow_mmap = TRUE; /* Default to using mmap(). */ - + /* GDBM_FAST used to determine whether or not we set fast_write. */ if (flags & GDBM_SYNC) { @@ -395,7 +397,9 @@ gdbm_open (const char *file, int block_size, int flags, int mode, #if HAVE_MMAP if (dbf->allow_mmap) { - if (_gdbm_mapped_init (dbf) == -1) + if (_gdbm_mapped_init (dbf) == 0) + dbf->mmap_inited = TRUE; + else { /* gdbm_errno should already be set. */ close (dbf->desc); @@ -424,7 +428,7 @@ gdbm_open (const char *file, int block_size, int flags, int mode, /* Initialize the bucket cache. */ int -_gdbm_init_cache(GDBM_FILE dbf, int size) +_gdbm_init_cache(GDBM_FILE dbf, size_t size) { int index; diff --git a/src/gdbmsetopt.c b/src/gdbmsetopt.c index 47f15fb..54a972f 100644 --- a/src/gdbmsetopt.c +++ b/src/gdbmsetopt.c @@ -23,10 +23,48 @@ /* operate on an already open descriptor. */ -/* ARGSUSED */ +static int +getbool (void *optval, int optlen) +{ + int n; + + if (!optval || optlen != sizeof (int) || + (((n = *(int*)optval) != TRUE) && n != FALSE)) + { + gdbm_errno = GDBM_OPT_ILLEGAL; + return -1; + } + return n; +} + +static int +get_size (void *optval, int optlen, size_t *ret) +{ + if (!optval) + { + gdbm_errno = GDBM_OPT_ILLEGAL; + return -1; + } + if (optlen == sizeof (unsigned)) + *ret = *(unsigned*) optval; + else if (optlen == sizeof (unsigned long)) + *ret = *(unsigned long*) optval; + else if (optlen == sizeof (size_t)) + *ret = *(size_t*) optval; + else + { + gdbm_errno = GDBM_OPT_ILLEGAL; + return -1; + } + return 0; +} + int -gdbm_setopt(GDBM_FILE dbf, int optflag, int *optval, int optlen) +gdbm_setopt (GDBM_FILE dbf, int optflag, void *optval, int optlen) { + int n; + size_t sz; + switch (optflag) { case GDBM_CACHESIZE: @@ -37,52 +75,50 @@ gdbm_setopt(GDBM_FILE dbf, int optflag, int *optval, int optlen) return -1; } - return _gdbm_init_cache(dbf, ((*optval) > 9) ? (*optval) : 10); + if (get_size (optval, optlen, &sz)) + return -1; + return _gdbm_init_cache (dbf, (sz > 9) ? sz : 10); case GDBM_FASTMODE: /* Obsolete form of SYNCMODE. */ - if (!optval || ((*optval != TRUE) && (*optval != FALSE))) - { - gdbm_errno = GDBM_OPT_ILLEGAL; - return -1; - } - - dbf->fast_write = *optval; + if ((n = getbool (optval, optlen)) == -1) + return -1; + dbf->fast_write = n; break; case GDBM_SYNCMODE: /* Optval will point to either true or false. */ - if (!optval || ((*optval != TRUE) && (*optval != FALSE))) - { - gdbm_errno = GDBM_OPT_ILLEGAL; - return -1; - } - - dbf->fast_write = !(*optval); + if ((n = getbool (optval, optlen)) == -1) + return -1; + dbf->fast_write = !n; break; case GDBM_CENTFREE: /* Optval will point to either true or false. */ - if (!optval || ((*optval != TRUE) && (*optval != FALSE))) - { - gdbm_errno = GDBM_OPT_ILLEGAL; - return -1; - } - - dbf->central_free = *optval; + if ((n = getbool (optval, optlen)) == -1) + return -1; + dbf->central_free = n; break; case GDBM_COALESCEBLKS: /* Optval will point to either true or false. */ - if (!optval || ((*optval != TRUE) && (*optval != FALSE))) - { - gdbm_errno = GDBM_OPT_ILLEGAL; - return -1; - } - - dbf->coalesce_blocks = *optval; + if ((n = getbool (optval, optlen)) == -1) + return -1; + dbf->coalesce_blocks = n; break; + case GDBM_SETMAXMAPSIZE: + { + size_t page_size = sysconf (_SC_PAGESIZE); + + if (get_size (optval, optlen, &sz)) + return -1; + dbf->mapped_size_max = ((sz + page_size - 1) / page_size) * + page_size; + _gdbm_mapped_init (dbf); + break; + } + default: gdbm_errno = GDBM_OPT_ILLEGAL; return -1; @@ -32,22 +32,19 @@ # define MAP_FAILED ((void*)-1) #endif -/* Maximum size representable by a size_t variable */ -#define SIZE_T_MAX ((size_t)-1) - /* Translate current offset in the mapped region into the absolute position */ #define _GDBM_MMAPPED_POS(dbf) ((dbf)->mapped_off + (dbf)->mapped_pos) /* Return true if the absolute offset OFF lies within the currentlty mmapped region */ #define _GDBM_IN_MAPPED_REGION_P(dbf, off) \ - ((off) > (dbf)->mapped_off \ + ((off) >= (dbf)->mapped_off \ && ((off) - (dbf)->mapped_off) < (dbf)->mapped_size) /* Return true if the current region needs to be remapped */ #define _GDBM_NEED_REMAP(dbf) \ - ((dbf)->mapped_remap || (dbf)->mapped_pos == (dbf)->mapped_size) + (!(dbf)->mapped_region || (dbf)->mapped_pos == (dbf)->mapped_size) /* Return the sum of the currently mapped size and DELTA */ #define SUM_FILE_SIZE(dbf, delta) \ - ((dbf)->mapped_size + delta) + ((dbf)->mapped_off + (dbf)->mapped_size + (delta)) /* Store the size of the GDBM file DBF in *PSIZE. Return 0 on success and -1 on failure. */ @@ -61,19 +58,17 @@ _gdbm_file_size (GDBM_FILE dbf, off_t *psize) return 0; } -/* Unmap the region. Reset all mapped_ fields to initial values. */ +/* Unmap the region. Reset all mapped fields to initial values. */ void _gdbm_mapped_unmap (GDBM_FILE dbf) { if (dbf->mapped_region) { - msync (dbf->mapped_region, 0, MS_SYNC | MS_INVALIDATE); munmap (dbf->mapped_region, dbf->mapped_size); dbf->mapped_region = NULL; dbf->mapped_size = 0; dbf->mapped_pos = 0; dbf->mapped_off = 0; - dbf->mapped_remap = 0; } } @@ -81,19 +76,25 @@ _gdbm_mapped_unmap (GDBM_FILE dbf) Take care to recompute {mapped_off,mapped_pos} so that the former lies on a page size boundary. */ int -_gdbm_internal_remap (GDBM_FILE dbf) +_gdbm_internal_remap (GDBM_FILE dbf, size_t size) { void *p; int flags = PROT_READ; size_t page_size = sysconf (_SC_PAGESIZE); + munmap (dbf->mapped_region, dbf->mapped_size); + dbf->mapped_size = size; + + if (size == 0) + return 0; + dbf->mapped_pos += dbf->mapped_off % page_size; dbf->mapped_off = (dbf->mapped_off / page_size) * page_size; if (dbf->read_write) flags |= PROT_WRITE; - p = mmap (dbf->mapped_region, dbf->mapped_size, flags, MAP_SHARED, + p = mmap (NULL, dbf->mapped_size, flags, MAP_SHARED, dbf->desc, dbf->mapped_off); if (p == MAP_FAILED) { @@ -103,23 +104,33 @@ _gdbm_internal_remap (GDBM_FILE dbf) } dbf->mapped_region = p; - dbf->mapped_remap = 0; return 0; } +#define _REMAP_DEFAULT 0 +#define _REMAP_EXTEND 1 +#define _REMAP_END 2 + /* Remap the GDBM file so that its mapped region ends on SIZEth byte. - If the file is opened with write permissions and EXTEND is not 0, - make sure the file is able to accomodate SIZE bytes. - Otherwise, trim SIZE to the actual size of the file. - Return 0 on success, -1 on failure. */ + If the file is opened with write permissions, FLAG controls how + it is expanded. The value _REMAP_DEFAULT truncates SIZE to the + actual file size. The value _REMAP_EXTEND extends the file, if + necessary, to accomodate max(SIZE,dbf->header->next_block) bytes. + Finally, the value _REMAP_END instructs the function to use + max(SIZE, file_size) as the upper bound of the mapped region. + + If the file is opened read-only, FLAG is ignored and SIZE is + truncated to the actual file size. + + The upper bound obtained that way is used as a *hint* to select + the actual size of the mapped region. which can never exceed + dbf->mapped_size_max. + + The function returns 0 on success, -1 on failure. */ int -_gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int extend) +_gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int flag) { - off_t file_size; - - if (dbf->mapped_region && !dbf->mapped_remap - && _GDBM_IN_MAPPED_REGION_P (dbf, size)) - return 0; + off_t file_size, pos; if (_gdbm_file_size (dbf, &file_size)) { @@ -128,16 +139,23 @@ _gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int extend) _gdbm_mapped_unmap (dbf); return -1; } + + if (flag == _REMAP_END && size < file_size) + size = file_size; if (dbf->read_write) { if (size > file_size) { - if (extend) + if (flag != _REMAP_DEFAULT) { char c = 0; + + if (size < dbf->header->next_block) + size = dbf->header->next_block; lseek (dbf->desc, size - 1, SEEK_SET); write (dbf->desc, &c, 1); + file_size = size; } else { @@ -155,35 +173,33 @@ _gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int extend) return 0; } - if (!dbf->mapped_remap) + pos = _GDBM_MMAPPED_POS (dbf); + if (size > dbf->mapped_size_max) { - if (sizeof (off_t) > sizeof (size_t) && size > SIZE_T_MAX) - { - off_t pos = _GDBM_MMAPPED_POS (dbf); - dbf->mapped_off = (size / SIZE_T_MAX) * SIZE_T_MAX; - size -= dbf->mapped_off; - if (pos < dbf->mapped_off) - pos = dbf->mapped_off; /* FIXME */ - dbf->mapped_pos = pos - dbf->mapped_off; - } + dbf->mapped_off = pos; + dbf->mapped_pos = 0; + size = dbf->mapped_size_max; + if (dbf->mapped_off + size > file_size) + size = file_size - dbf->mapped_off; } - dbf->mapped_size = size; - return _gdbm_internal_remap (dbf); + else + { + dbf->mapped_pos += dbf->mapped_off; + dbf->mapped_off = 0; + } + + return _gdbm_internal_remap (dbf, size); } -/* Initialize mapping system. If the file size is less than SIZE_T_MAX, - map the entire file into the memory. Otherwise, map first SIZE_T_MAX +/* Initialize mapping system. If the file size is less than MAPPED_SIZE_MAX, + map the entire file into the memory. Otherwise, map first MAPPED_SIZE_MAX bytes. */ int _gdbm_mapped_init (GDBM_FILE dbf) { - off_t file_size; - - if (_gdbm_file_size (dbf, &file_size)) - return -1; - if (file_size > SIZE_T_MAX) - file_size = SIZE_T_MAX; - return _gdbm_mapped_remap (dbf, file_size, 1); + if (dbf->mapped_size_max == 0) + dbf->mapped_size_max = SIZE_T_MAX; + return _gdbm_mapped_remap (dbf, 0, _REMAP_END); } /* Read LEN bytes from the GDBM file DBF into BUFFER. If mmapping is @@ -192,7 +208,7 @@ _gdbm_mapped_init (GDBM_FILE dbf) ssize_t _gdbm_mapped_read (GDBM_FILE dbf, void *buffer, size_t len) { - if (dbf->mapped_region) + if (dbf->mmap_inited) { ssize_t total = 0; char *cbuf = buffer; @@ -204,9 +220,12 @@ _gdbm_mapped_read (GDBM_FILE dbf, void *buffer, size_t len) if (_GDBM_NEED_REMAP (dbf)) { off_t pos = _GDBM_MMAPPED_POS (dbf); - if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len), 0)) + if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len), + _REMAP_DEFAULT)) { int rc; + + dbf->mmap_inited = FALSE; if (lseek (dbf->desc, pos, SEEK_SET) != pos) return total > 0 ? total : -1; rc = read (dbf->desc, cbuf, len); @@ -239,11 +258,11 @@ _gdbm_mapped_read (GDBM_FILE dbf, void *buffer, size_t len) ssize_t _gdbm_mapped_write (GDBM_FILE dbf, void *buffer, size_t len) { - if (dbf->mapped_region) + if (dbf->mmap_inited) { ssize_t total = 0; char *cbuf = buffer; - + while (len) { size_t nbytes; @@ -251,9 +270,12 @@ _gdbm_mapped_write (GDBM_FILE dbf, void *buffer, size_t len) if (_GDBM_NEED_REMAP (dbf)) { off_t pos = _GDBM_MMAPPED_POS (dbf); - if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len), 1)) + if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len), + _REMAP_EXTEND)) { int rc; + + dbf->mmap_inited = FALSE; if (lseek (dbf->desc, pos, SEEK_SET) != pos) return total > 0 ? total : -1; rc = write (dbf->desc, cbuf, len); @@ -290,7 +312,7 @@ _gdbm_mapped_write (GDBM_FILE dbf, void *buffer, size_t len) off_t _gdbm_mapped_lseek (GDBM_FILE dbf, off_t offset, int whence) { - if (dbf->mapped_region) + if (dbf->mmap_inited) { off_t needle; @@ -320,17 +342,12 @@ _gdbm_mapped_lseek (GDBM_FILE dbf, off_t offset, int whence) if (!_GDBM_IN_MAPPED_REGION_P (dbf, needle)) { - dbf->mapped_remap = 1; - if (needle > dbf->mapped_size) - dbf->mapped_size = needle; - if (sizeof (off_t) > sizeof (size_t) && needle > SIZE_T_MAX) - dbf->mapped_off = (needle / SIZE_T_MAX) * SIZE_T_MAX; - else - dbf->mapped_off = 0; + _gdbm_mapped_unmap (dbf); + dbf->mapped_off = needle; + dbf->mapped_pos = 0; } - - dbf->mapped_pos = needle - dbf->mapped_off; - + else + dbf->mapped_pos = needle - dbf->mapped_off; return needle; } return lseek (dbf->desc, offset, whence); @@ -338,14 +355,14 @@ _gdbm_mapped_lseek (GDBM_FILE dbf, off_t offset, int whence) /* Sync the mapped region to disk. */ int -_gdbm_mapped_sync(GDBM_FILE dbf) +_gdbm_mapped_sync (GDBM_FILE dbf) { if (dbf->mapped_region) { - return (msync(dbf->mapped_region, dbf->mapped_size, - MS_SYNC | MS_INVALIDATE)); + return msync (dbf->mapped_region, dbf->mapped_size, + MS_SYNC | MS_INVALIDATE); } - return (fsync(dbf->desc)); + return fsync (dbf->desc); } #endif diff --git a/src/proto.h b/src/proto.h index db3f698..de96878 100644 --- a/src/proto.h +++ b/src/proto.h @@ -41,7 +41,7 @@ void _gdbm_end_update (GDBM_FILE); void _gdbm_fatal (GDBM_FILE, const char *); /* From gdbmopen.c */ -int _gdbm_init_cache (GDBM_FILE, int); +int _gdbm_init_cache (GDBM_FILE, size_t); /* From mmap.c */ int _gdbm_mapped_init (GDBM_FILE); diff --git a/src/update.c b/src/update.c index aa9378f..b814415 100644 --- a/src/update.c +++ b/src/update.c @@ -38,8 +38,8 @@ write_header (GDBM_FILE dbf) if (num_bytes != dbf->header->block_size) _gdbm_fatal (dbf, "write error"); - /* Sync the file if fast_write is FALSE or it's mapped. */ - if (dbf->mapped_region != NULL || dbf->fast_write == FALSE) + /* Sync the file if fast_write is FALSE. */ + if (dbf->fast_write == FALSE) __fsync (dbf); } @@ -85,8 +85,7 @@ _gdbm_end_update (GDBM_FILE dbf) if (num_bytes != dbf->header->dir_size) _gdbm_fatal (dbf, "write error"); dbf->directory_changed = FALSE; - if (!dbf->header_changed && - (dbf->fast_write == FALSE || dbf->mapped_region != NULL)) + if (!dbf->header_changed && dbf->fast_write == FALSE) __fsync (dbf); } diff --git a/tests/gtload.c b/tests/gtload.c index 550833f..b89ae1f 100644 --- a/tests/gtload.c +++ b/tests/gtload.c @@ -18,6 +18,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <errno.h> #include "gdbm.h" #include "progname.h" @@ -37,6 +38,7 @@ main (int argc, char **argv) GDBM_FILE dbf; int delim = '\t'; int data_z = 0; + size_t mapped_size_max = 0; while (--argc) { @@ -44,7 +46,7 @@ main (int argc, char **argv) if (strcmp (arg, "-h") == 0) { - printf ("usage: %s [-replace] [-clear] [-blocksize=N] [-null] [-nolock] [-nommap] [-sync] [-delim=CHR] DBFILE\n", progname); + printf ("usage: %s [-replace] [-clear] [-blocksize=N] [-null] [-nolock] [-nommap] [-maxmap=N] [-sync] [-delim=CHR] DBFILE\n", progname); exit (0); } else if (strcmp (arg, "-replace") == 0) @@ -61,6 +63,26 @@ main (int argc, char **argv) flags |= GDBM_SYNC; else if (strncmp (arg, "-blocksize=", 11) == 0) block_size = atoi (arg + 11); + else if (strncmp (arg, "-maxmap=", 8) == 0) + { + char *p; + + errno = 0; + mapped_size_max = strtoul (arg + 8, &p, 10); + + if (errno) + { + fprintf (stderr, "%s: ", progname); + perror ("maxmap"); + exit (1); + } + + if (*p) + { + fprintf (stderr, "%s: bad maxmap\n", progname); + exit (1); + } + } else if (strncmp (arg, "-delim=", 7) == 0) delim = arg[7]; else if (strcmp (arg, "--") == 0) @@ -92,6 +114,17 @@ main (int argc, char **argv) exit (1); } + if (mapped_size_max) + { + if (gdbm_setopt (dbf, GDBM_SETMAXMAPSIZE, &mapped_size_max, + sizeof (mapped_size_max))) + { + fprintf (stderr, "gdbm_setopt failed: %s\n", + gdbm_strerror (gdbm_errno)); + exit (1); + } + } + while (fgets (buf, sizeof buf, stdin)) { size_t i, j; |