aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org.ua>2011-08-10 22:00:34 +0000
committerSergey Poznyakoff <gray@gnu.org.ua>2011-08-10 22:00:34 +0000
commit74f7fe561e31637bdbf21aeb817d4a22bbf9c91a (patch)
treebae9da9f2c749d290ad4f1764dc205bcd23b750e
parent6b6dcc80555d77858733e44b72e92d1dc60e2c38 (diff)
downloadgdbm-74f7fe561e31637bdbf21aeb817d4a22bbf9c91a.tar.gz
gdbm-74f7fe561e31637bdbf21aeb817d4a22bbf9c91a.tar.bz2
Improve memory mapping support.
The new code is more flexible and performs better when lots of inserts are being made (e.g. when populating the database with new data). * src/gdbm.h.in (GDBM_SETMAXMAPSIZE): New constant. * src/gdbmconst.h (SIZE_T_MAX): New define. * src/gdbmdefs.h (gdbm_file_info) <cache_size>: Change type to size_t. <mmap_inited,mapped_size_max>: New member. <mapped_remap>: Remove. * src/gdbmopen.c: Fix a typo. (gdbm_open): Initialize new members. (_gdbm_init_cache): Second argument is size_t. * src/gdbmsetopt.c (gdbm_setopt): Optval argument is void*. Handle GDBM_SETMAXMAPSIZE. Improve error checking. * src/mmap.c (_GDBM_IN_MAPPED_REGION_P): Fix comparison with the lower bound. (_GDBM_NEED_REMAP): Return true if mapped_region is NULL. (SUM_FILE_SIZE): Rewrite. (_gdbm_mapped_unmap): Don't call msync. (_gdbm_internal_remap): Take 2 arguments, the second one giving the new mapped size. Unmap the region prior to remapping it. Always pass NULL as the argument to mmap. (_gdbm_mapped_remap): Rewrite the logic. Change semantics of the third argument. All uses updated. (_gdbm_mapped_init): Reflect the above changes. (_gdbm_mapped_read,_gdbm_mapped_write): Use mmap_inited to decide whether to use mmap, because mapped_region can be reset to zero by another functions (namely, _gdbm_mapped_lseek). Reset mmap_inited to FALSE, if _gdbm_mapped_remap fails. (_gdbm_mapped_lseek): Rewrite offset computations. Invalidate the mapped region. * src/proto.h (_gdbm_init_cache): Change prototype. * src/update.c (write_header, _gdbm_end_update): Remove checks for dbf->mapped_region. * tests/gtload.c: Implement the -maxmap option (set maximal mapped memory size). * doc/gdbm.texinfo: Document GDBM_SETMAXMAPSIZE.
-rw-r--r--ChangeLog46
-rw-r--r--doc/gdbm.texinfo10
-rw-r--r--src/gdbm.h.in4
-rw-r--r--src/gdbmconst.h3
-rw-r--r--src/gdbmdefs.h12
-rw-r--r--src/gdbmopen.c12
-rw-r--r--src/gdbmsetopt.c98
-rw-r--r--src/mmap.c145
-rw-r--r--src/proto.h2
-rw-r--r--src/update.c7
-rw-r--r--tests/gtload.c35
11 files changed, 259 insertions, 115 deletions
diff --git a/ChangeLog b/ChangeLog
index 4095ab0..1b358b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,49 @@
+2011-08-11 Sergey Poznyakoff <gray@gnu.org.ua>
+
+ Improve memory mapping support.
+
+ The new code is more flexible and performs better when
+ lots of inserts are being made (e.g. when populating the
+ database with new data).
+
+ * src/gdbm.h.in (GDBM_SETMAXMAPSIZE): New constant.
+ * src/gdbmconst.h (SIZE_T_MAX): New define.
+ * src/gdbmdefs.h (gdbm_file_info) <cache_size>: Change type
+ to size_t.
+ <mmap_inited,mapped_size_max>: New member.
+ <mapped_remap>: Remove.
+ * src/gdbmopen.c: Fix a typo.
+ (gdbm_open): Initialize new members.
+ (_gdbm_init_cache): Second argument is size_t.
+ * src/gdbmsetopt.c (gdbm_setopt): Optval argument is void*.
+ Handle GDBM_SETMAXMAPSIZE.
+ Improve error checking.
+ * src/mmap.c (_GDBM_IN_MAPPED_REGION_P): Fix comparison with
+ the lower bound.
+ (_GDBM_NEED_REMAP): Return true if mapped_region is NULL.
+ (SUM_FILE_SIZE): Rewrite.
+ (_gdbm_mapped_unmap): Don't call msync.
+ (_gdbm_internal_remap): Take 2 arguments, the second one
+ giving the new mapped size.
+ Unmap the region prior to remapping it.
+ Always pass NULL as the argument to mmap.
+ (_gdbm_mapped_remap): Rewrite the logic. Change semantics of the
+ third argument. All uses updated.
+ (_gdbm_mapped_init): Reflect the above changes.
+ (_gdbm_mapped_read,_gdbm_mapped_write): Use mmap_inited to decide
+ whether to use mmap, because mapped_region can be reset to zero
+ by another functions (namely, _gdbm_mapped_lseek).
+ Reset mmap_inited to FALSE, if _gdbm_mapped_remap fails.
+ (_gdbm_mapped_lseek): Rewrite offset computations. Invalidate
+ the mapped region.
+ * src/proto.h (_gdbm_init_cache): Change prototype.
+ * src/update.c (write_header, _gdbm_end_update): Remove checks
+ for dbf->mapped_region.
+ * tests/gtload.c: Implement the -maxmap option (set maximal
+ mapped memory size).
+
+ * doc/gdbm.texinfo: Document GDBM_SETMAXMAPSIZE.
+
2011-08-09 Sergey Poznyakoff <gray@gnu.org.ua>
Update the docs.
diff --git a/doc/gdbm.texinfo b/doc/gdbm.texinfo
index c349941..5dc8a97 100644
--- a/doc/gdbm.texinfo
+++ b/doc/gdbm.texinfo
@@ -738,7 +738,7 @@ The @var{errno} argument is usually the value of the global variable
open database.
@deftypefn {gdbm interface} int gdbm_setopt (GDBM_FILE @var{dbf}, int @var{option}, @
- int *@var{value}, int @var{size})
+ void *@var{value}, int @var{size})
Sets an option on the database.
The parameters are:
@@ -809,6 +809,14 @@ a @acronym{CPU} expensive process with time, though, especially if
used in conjunction with GDBM_CENTFREE. The @var{value} should point
to an integer: @samp{TRUE} to turn free block merging on, and @samp{FALSE} to
turn it off.
+
+@kwindex GDBM_SETMAXMAPSIZE
+@item GDBM_SETMAXMAPSIZE
+Sets maximum size of the memory mapped region. The @var{value} should
+point to a value of type @code{size_t}, @code{unsigned long} or
+@code{unsigned}. The actual value is rounded to the nearest page
+boundary (the page size is obtained from
+@code{sysconf(_SC_PAGESIZE)}).
@end table
The return value will be @samp{-1} upon failure, or @samp{0} upon
diff --git a/src/gdbm.h.in b/src/gdbm.h.in
index 70de4e0..8eaee79 100644
--- a/src/gdbm.h.in
+++ b/src/gdbm.h.in
@@ -51,6 +51,7 @@
#define GDBM_SYNCMODE 3 /* Turn on or off sync operations. */
#define GDBM_CENTFREE 4 /* Keep all free blocks in the header. */
#define GDBM_COALESCEBLKS 5 /* Attempt to coalesce free blocks. */
+#define GDBM_SETMAXMAPSIZE 6 /* Set maximum mapped memory size */
/* The data and key structure. */
typedef struct {
@@ -89,7 +90,7 @@ extern datum gdbm_nextkey (GDBM_FILE, datum);
extern int gdbm_reorganize (GDBM_FILE);
extern void gdbm_sync (GDBM_FILE);
extern int gdbm_exists (GDBM_FILE, datum);
-extern int gdbm_setopt (GDBM_FILE, int, int *, int);
+extern int gdbm_setopt (GDBM_FILE, int, void *, int);
extern int gdbm_fdesc (GDBM_FILE);
extern int gdbm_export (GDBM_FILE, const char *, int, int);
extern int gdbm_import (GDBM_FILE, const char *, int);
@@ -136,6 +137,7 @@ extern "C" {
#endif
extern const char *gdbm_strerror (gdbm_error);
+extern int gdbm_version_cmp (int const a[], int const b[]);
#if defined(__cplusplus) || defined(c_plusplus)
}
diff --git a/src/gdbmconst.h b/src/gdbmconst.h
index a4c8cfd..7a3f5b9 100644
--- a/src/gdbmconst.h
+++ b/src/gdbmconst.h
@@ -70,3 +70,6 @@
/* The size of the bucket cache. */
#define DEFAULT_CACHESIZE 100
+
+/* Maximum size representable by a size_t variable */
+#define SIZE_T_MAX ((size_t)-1)
diff --git a/src/gdbmdefs.h b/src/gdbmdefs.h
index 819abbc..2221aa5 100644
--- a/src/gdbmdefs.h
+++ b/src/gdbmdefs.h
@@ -124,8 +124,6 @@ typedef struct {
data_cache_elem ca_data;
} cache_elem;
-
-
/* This final structure contains all main memory based information for
a gdbm file. This allows multiple gdbm files to be opened at the same
time by one program. */
@@ -153,7 +151,8 @@ struct gdbm_file_info {
/* Whether or not we're allowing mmap() use. */
unsigned allow_mmap :1;
-
+ unsigned mmap_inited :1;
+
/* Type of file locking in use. */
enum { LOCKING_NONE = 0, LOCKING_FLOCK, LOCKING_LOCKF,
LOCKING_FCNTL } lock_type;
@@ -173,7 +172,7 @@ struct gdbm_file_info {
/* The bucket cache. */
cache_elem *bucket_cache;
- int cache_size;
+ size_t cache_size;
int last_read;
/* Points to the current hash bucket in the cache. */
@@ -193,15 +192,12 @@ struct gdbm_file_info {
unsigned second_changed :1;
/* Mmap info */
+ size_t mapped_size_max;/* Max. allowed value for mapped_size */
void *mapped_region; /* Mapped region */
size_t mapped_size; /* Size of the region */
off_t mapped_pos; /* Current offset in the region */
off_t mapped_off; /* Position in the file where the region
begins */
- int mapped_remap; /* When set, any call to
- _gdbm_mapped_{write|read} will remap the
- region according to the above fields. */
-
};
/* Now define all the routines in use. */
diff --git a/src/gdbmopen.c b/src/gdbmopen.c
index 7cd9040..a110d64 100644
--- a/src/gdbmopen.c
+++ b/src/gdbmopen.c
@@ -28,7 +28,7 @@
#elif SIZEOF_OFF_T == 8
#define GDBM_MAGIC GDBM_MAGIC64
#elif
-#error "Unsupport off_t size, contact GDBM maintainer. What crazy system is this?!?"
+#error "Unsupported off_t size, contact GDBM maintainer. What crazy system is this?!?"
#endif
/* Initialize dbm system. FILE is a pointer to the file name. If the file
@@ -84,6 +84,8 @@ gdbm_open (const char *file, int block_size, int flags, int mode,
dbf->bucket_cache = NULL;
dbf->cache_size = 0;
+ dbf->mmap_inited = FALSE;
+ dbf->mapped_size_max = SIZE_T_MAX;
dbf->mapped_region = NULL;
dbf->mapped_size = 0;
dbf->mapped_pos = 0;
@@ -108,7 +110,7 @@ gdbm_open (const char *file, int block_size, int flags, int mode,
dbf->central_free = FALSE; /* Default to not using central_free. */
dbf->coalesce_blocks = FALSE; /* Default to not coalescing blocks. */
dbf->allow_mmap = TRUE; /* Default to using mmap(). */
-
+
/* GDBM_FAST used to determine whether or not we set fast_write. */
if (flags & GDBM_SYNC)
{
@@ -395,7 +397,9 @@ gdbm_open (const char *file, int block_size, int flags, int mode,
#if HAVE_MMAP
if (dbf->allow_mmap)
{
- if (_gdbm_mapped_init (dbf) == -1)
+ if (_gdbm_mapped_init (dbf) == 0)
+ dbf->mmap_inited = TRUE;
+ else
{
/* gdbm_errno should already be set. */
close (dbf->desc);
@@ -424,7 +428,7 @@ gdbm_open (const char *file, int block_size, int flags, int mode,
/* Initialize the bucket cache. */
int
-_gdbm_init_cache(GDBM_FILE dbf, int size)
+_gdbm_init_cache(GDBM_FILE dbf, size_t size)
{
int index;
diff --git a/src/gdbmsetopt.c b/src/gdbmsetopt.c
index 47f15fb..54a972f 100644
--- a/src/gdbmsetopt.c
+++ b/src/gdbmsetopt.c
@@ -23,10 +23,48 @@
/* operate on an already open descriptor. */
-/* ARGSUSED */
+static int
+getbool (void *optval, int optlen)
+{
+ int n;
+
+ if (!optval || optlen != sizeof (int) ||
+ (((n = *(int*)optval) != TRUE) && n != FALSE))
+ {
+ gdbm_errno = GDBM_OPT_ILLEGAL;
+ return -1;
+ }
+ return n;
+}
+
+static int
+get_size (void *optval, int optlen, size_t *ret)
+{
+ if (!optval)
+ {
+ gdbm_errno = GDBM_OPT_ILLEGAL;
+ return -1;
+ }
+ if (optlen == sizeof (unsigned))
+ *ret = *(unsigned*) optval;
+ else if (optlen == sizeof (unsigned long))
+ *ret = *(unsigned long*) optval;
+ else if (optlen == sizeof (size_t))
+ *ret = *(size_t*) optval;
+ else
+ {
+ gdbm_errno = GDBM_OPT_ILLEGAL;
+ return -1;
+ }
+ return 0;
+}
+
int
-gdbm_setopt(GDBM_FILE dbf, int optflag, int *optval, int optlen)
+gdbm_setopt (GDBM_FILE dbf, int optflag, void *optval, int optlen)
{
+ int n;
+ size_t sz;
+
switch (optflag)
{
case GDBM_CACHESIZE:
@@ -37,52 +75,50 @@ gdbm_setopt(GDBM_FILE dbf, int optflag, int *optval, int optlen)
return -1;
}
- return _gdbm_init_cache(dbf, ((*optval) > 9) ? (*optval) : 10);
+ if (get_size (optval, optlen, &sz))
+ return -1;
+ return _gdbm_init_cache (dbf, (sz > 9) ? sz : 10);
case GDBM_FASTMODE:
/* Obsolete form of SYNCMODE. */
- if (!optval || ((*optval != TRUE) && (*optval != FALSE)))
- {
- gdbm_errno = GDBM_OPT_ILLEGAL;
- return -1;
- }
-
- dbf->fast_write = *optval;
+ if ((n = getbool (optval, optlen)) == -1)
+ return -1;
+ dbf->fast_write = n;
break;
case GDBM_SYNCMODE:
/* Optval will point to either true or false. */
- if (!optval || ((*optval != TRUE) && (*optval != FALSE)))
- {
- gdbm_errno = GDBM_OPT_ILLEGAL;
- return -1;
- }
-
- dbf->fast_write = !(*optval);
+ if ((n = getbool (optval, optlen)) == -1)
+ return -1;
+ dbf->fast_write = !n;
break;
case GDBM_CENTFREE:
/* Optval will point to either true or false. */
- if (!optval || ((*optval != TRUE) && (*optval != FALSE)))
- {
- gdbm_errno = GDBM_OPT_ILLEGAL;
- return -1;
- }
-
- dbf->central_free = *optval;
+ if ((n = getbool (optval, optlen)) == -1)
+ return -1;
+ dbf->central_free = n;
break;
case GDBM_COALESCEBLKS:
/* Optval will point to either true or false. */
- if (!optval || ((*optval != TRUE) && (*optval != FALSE)))
- {
- gdbm_errno = GDBM_OPT_ILLEGAL;
- return -1;
- }
-
- dbf->coalesce_blocks = *optval;
+ if ((n = getbool (optval, optlen)) == -1)
+ return -1;
+ dbf->coalesce_blocks = n;
break;
+ case GDBM_SETMAXMAPSIZE:
+ {
+ size_t page_size = sysconf (_SC_PAGESIZE);
+
+ if (get_size (optval, optlen, &sz))
+ return -1;
+ dbf->mapped_size_max = ((sz + page_size - 1) / page_size) *
+ page_size;
+ _gdbm_mapped_init (dbf);
+ break;
+ }
+
default:
gdbm_errno = GDBM_OPT_ILLEGAL;
return -1;
diff --git a/src/mmap.c b/src/mmap.c
index 5348db9..2107bd6 100644
--- a/src/mmap.c
+++ b/src/mmap.c
@@ -32,22 +32,19 @@
# define MAP_FAILED ((void*)-1)
#endif
-/* Maximum size representable by a size_t variable */
-#define SIZE_T_MAX ((size_t)-1)
-
/* Translate current offset in the mapped region into the absolute position */
#define _GDBM_MMAPPED_POS(dbf) ((dbf)->mapped_off + (dbf)->mapped_pos)
/* Return true if the absolute offset OFF lies within the currentlty mmapped
region */
#define _GDBM_IN_MAPPED_REGION_P(dbf, off) \
- ((off) > (dbf)->mapped_off \
+ ((off) >= (dbf)->mapped_off \
&& ((off) - (dbf)->mapped_off) < (dbf)->mapped_size)
/* Return true if the current region needs to be remapped */
#define _GDBM_NEED_REMAP(dbf) \
- ((dbf)->mapped_remap || (dbf)->mapped_pos == (dbf)->mapped_size)
+ (!(dbf)->mapped_region || (dbf)->mapped_pos == (dbf)->mapped_size)
/* Return the sum of the currently mapped size and DELTA */
#define SUM_FILE_SIZE(dbf, delta) \
- ((dbf)->mapped_size + delta)
+ ((dbf)->mapped_off + (dbf)->mapped_size + (delta))
/* Store the size of the GDBM file DBF in *PSIZE.
Return 0 on success and -1 on failure. */
@@ -61,19 +58,17 @@ _gdbm_file_size (GDBM_FILE dbf, off_t *psize)
return 0;
}
-/* Unmap the region. Reset all mapped_ fields to initial values. */
+/* Unmap the region. Reset all mapped fields to initial values. */
void
_gdbm_mapped_unmap (GDBM_FILE dbf)
{
if (dbf->mapped_region)
{
- msync (dbf->mapped_region, 0, MS_SYNC | MS_INVALIDATE);
munmap (dbf->mapped_region, dbf->mapped_size);
dbf->mapped_region = NULL;
dbf->mapped_size = 0;
dbf->mapped_pos = 0;
dbf->mapped_off = 0;
- dbf->mapped_remap = 0;
}
}
@@ -81,19 +76,25 @@ _gdbm_mapped_unmap (GDBM_FILE dbf)
Take care to recompute {mapped_off,mapped_pos} so that the former lies
on a page size boundary. */
int
-_gdbm_internal_remap (GDBM_FILE dbf)
+_gdbm_internal_remap (GDBM_FILE dbf, size_t size)
{
void *p;
int flags = PROT_READ;
size_t page_size = sysconf (_SC_PAGESIZE);
+ munmap (dbf->mapped_region, dbf->mapped_size);
+ dbf->mapped_size = size;
+
+ if (size == 0)
+ return 0;
+
dbf->mapped_pos += dbf->mapped_off % page_size;
dbf->mapped_off = (dbf->mapped_off / page_size) * page_size;
if (dbf->read_write)
flags |= PROT_WRITE;
- p = mmap (dbf->mapped_region, dbf->mapped_size, flags, MAP_SHARED,
+ p = mmap (NULL, dbf->mapped_size, flags, MAP_SHARED,
dbf->desc, dbf->mapped_off);
if (p == MAP_FAILED)
{
@@ -103,23 +104,33 @@ _gdbm_internal_remap (GDBM_FILE dbf)
}
dbf->mapped_region = p;
- dbf->mapped_remap = 0;
return 0;
}
+#define _REMAP_DEFAULT 0
+#define _REMAP_EXTEND 1
+#define _REMAP_END 2
+
/* Remap the GDBM file so that its mapped region ends on SIZEth byte.
- If the file is opened with write permissions and EXTEND is not 0,
- make sure the file is able to accomodate SIZE bytes.
- Otherwise, trim SIZE to the actual size of the file.
- Return 0 on success, -1 on failure. */
+ If the file is opened with write permissions, FLAG controls how
+ it is expanded. The value _REMAP_DEFAULT truncates SIZE to the
+ actual file size. The value _REMAP_EXTEND extends the file, if
+ necessary, to accomodate max(SIZE,dbf->header->next_block) bytes.
+ Finally, the value _REMAP_END instructs the function to use
+ max(SIZE, file_size) as the upper bound of the mapped region.
+
+ If the file is opened read-only, FLAG is ignored and SIZE is
+ truncated to the actual file size.
+
+ The upper bound obtained that way is used as a *hint* to select
+ the actual size of the mapped region. which can never exceed
+ dbf->mapped_size_max.
+
+ The function returns 0 on success, -1 on failure. */
int
-_gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int extend)
+_gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int flag)
{
- off_t file_size;
-
- if (dbf->mapped_region && !dbf->mapped_remap
- && _GDBM_IN_MAPPED_REGION_P (dbf, size))
- return 0;
+ off_t file_size, pos;
if (_gdbm_file_size (dbf, &file_size))
{
@@ -128,16 +139,23 @@ _gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int extend)
_gdbm_mapped_unmap (dbf);
return -1;
}
+
+ if (flag == _REMAP_END && size < file_size)
+ size = file_size;
if (dbf->read_write)
{
if (size > file_size)
{
- if (extend)
+ if (flag != _REMAP_DEFAULT)
{
char c = 0;
+
+ if (size < dbf->header->next_block)
+ size = dbf->header->next_block;
lseek (dbf->desc, size - 1, SEEK_SET);
write (dbf->desc, &c, 1);
+ file_size = size;
}
else
{
@@ -155,35 +173,33 @@ _gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int extend)
return 0;
}
- if (!dbf->mapped_remap)
+ pos = _GDBM_MMAPPED_POS (dbf);
+ if (size > dbf->mapped_size_max)
{
- if (sizeof (off_t) > sizeof (size_t) && size > SIZE_T_MAX)
- {
- off_t pos = _GDBM_MMAPPED_POS (dbf);
- dbf->mapped_off = (size / SIZE_T_MAX) * SIZE_T_MAX;
- size -= dbf->mapped_off;
- if (pos < dbf->mapped_off)
- pos = dbf->mapped_off; /* FIXME */
- dbf->mapped_pos = pos - dbf->mapped_off;
- }
+ dbf->mapped_off = pos;
+ dbf->mapped_pos = 0;
+ size = dbf->mapped_size_max;
+ if (dbf->mapped_off + size > file_size)
+ size = file_size - dbf->mapped_off;
}
- dbf->mapped_size = size;
- return _gdbm_internal_remap (dbf);
+ else
+ {
+ dbf->mapped_pos += dbf->mapped_off;
+ dbf->mapped_off = 0;
+ }
+
+ return _gdbm_internal_remap (dbf, size);
}
-/* Initialize mapping system. If the file size is less than SIZE_T_MAX,
- map the entire file into the memory. Otherwise, map first SIZE_T_MAX
+/* Initialize mapping system. If the file size is less than MAPPED_SIZE_MAX,
+ map the entire file into the memory. Otherwise, map first MAPPED_SIZE_MAX
bytes. */
int
_gdbm_mapped_init (GDBM_FILE dbf)
{
- off_t file_size;
-
- if (_gdbm_file_size (dbf, &file_size))
- return -1;
- if (file_size > SIZE_T_MAX)
- file_size = SIZE_T_MAX;
- return _gdbm_mapped_remap (dbf, file_size, 1);
+ if (dbf->mapped_size_max == 0)
+ dbf->mapped_size_max = SIZE_T_MAX;
+ return _gdbm_mapped_remap (dbf, 0, _REMAP_END);
}
/* Read LEN bytes from the GDBM file DBF into BUFFER. If mmapping is
@@ -192,7 +208,7 @@ _gdbm_mapped_init (GDBM_FILE dbf)
ssize_t
_gdbm_mapped_read (GDBM_FILE dbf, void *buffer, size_t len)
{
- if (dbf->mapped_region)
+ if (dbf->mmap_inited)
{
ssize_t total = 0;
char *cbuf = buffer;
@@ -204,9 +220,12 @@ _gdbm_mapped_read (GDBM_FILE dbf, void *buffer, size_t len)
if (_GDBM_NEED_REMAP (dbf))
{
off_t pos = _GDBM_MMAPPED_POS (dbf);
- if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len), 0))
+ if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len),
+ _REMAP_DEFAULT))
{
int rc;
+
+ dbf->mmap_inited = FALSE;
if (lseek (dbf->desc, pos, SEEK_SET) != pos)
return total > 0 ? total : -1;
rc = read (dbf->desc, cbuf, len);
@@ -239,11 +258,11 @@ _gdbm_mapped_read (GDBM_FILE dbf, void *buffer, size_t len)
ssize_t
_gdbm_mapped_write (GDBM_FILE dbf, void *buffer, size_t len)
{
- if (dbf->mapped_region)
+ if (dbf->mmap_inited)
{
ssize_t total = 0;
char *cbuf = buffer;
-
+
while (len)
{
size_t nbytes;
@@ -251,9 +270,12 @@ _gdbm_mapped_write (GDBM_FILE dbf, void *buffer, size_t len)
if (_GDBM_NEED_REMAP (dbf))
{
off_t pos = _GDBM_MMAPPED_POS (dbf);
- if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len), 1))
+ if (_gdbm_mapped_remap (dbf, SUM_FILE_SIZE (dbf, len),
+ _REMAP_EXTEND))
{
int rc;
+
+ dbf->mmap_inited = FALSE;
if (lseek (dbf->desc, pos, SEEK_SET) != pos)
return total > 0 ? total : -1;
rc = write (dbf->desc, cbuf, len);
@@ -290,7 +312,7 @@ _gdbm_mapped_write (GDBM_FILE dbf, void *buffer, size_t len)
off_t
_gdbm_mapped_lseek (GDBM_FILE dbf, off_t offset, int whence)
{
- if (dbf->mapped_region)
+ if (dbf->mmap_inited)
{
off_t needle;
@@ -320,17 +342,12 @@ _gdbm_mapped_lseek (GDBM_FILE dbf, off_t offset, int whence)
if (!_GDBM_IN_MAPPED_REGION_P (dbf, needle))
{
- dbf->mapped_remap = 1;
- if (needle > dbf->mapped_size)
- dbf->mapped_size = needle;
- if (sizeof (off_t) > sizeof (size_t) && needle > SIZE_T_MAX)
- dbf->mapped_off = (needle / SIZE_T_MAX) * SIZE_T_MAX;
- else
- dbf->mapped_off = 0;
+ _gdbm_mapped_unmap (dbf);
+ dbf->mapped_off = needle;
+ dbf->mapped_pos = 0;
}
-
- dbf->mapped_pos = needle - dbf->mapped_off;
-
+ else
+ dbf->mapped_pos = needle - dbf->mapped_off;
return needle;
}
return lseek (dbf->desc, offset, whence);
@@ -338,14 +355,14 @@ _gdbm_mapped_lseek (GDBM_FILE dbf, off_t offset, int whence)
/* Sync the mapped region to disk. */
int
-_gdbm_mapped_sync(GDBM_FILE dbf)
+_gdbm_mapped_sync (GDBM_FILE dbf)
{
if (dbf->mapped_region)
{
- return (msync(dbf->mapped_region, dbf->mapped_size,
- MS_SYNC | MS_INVALIDATE));
+ return msync (dbf->mapped_region, dbf->mapped_size,
+ MS_SYNC | MS_INVALIDATE);
}
- return (fsync(dbf->desc));
+ return fsync (dbf->desc);
}
#endif
diff --git a/src/proto.h b/src/proto.h
index db3f698..de96878 100644
--- a/src/proto.h
+++ b/src/proto.h
@@ -41,7 +41,7 @@ void _gdbm_end_update (GDBM_FILE);
void _gdbm_fatal (GDBM_FILE, const char *);
/* From gdbmopen.c */
-int _gdbm_init_cache (GDBM_FILE, int);
+int _gdbm_init_cache (GDBM_FILE, size_t);
/* From mmap.c */
int _gdbm_mapped_init (GDBM_FILE);
diff --git a/src/update.c b/src/update.c
index aa9378f..b814415 100644
--- a/src/update.c
+++ b/src/update.c
@@ -38,8 +38,8 @@ write_header (GDBM_FILE dbf)
if (num_bytes != dbf->header->block_size)
_gdbm_fatal (dbf, "write error");
- /* Sync the file if fast_write is FALSE or it's mapped. */
- if (dbf->mapped_region != NULL || dbf->fast_write == FALSE)
+ /* Sync the file if fast_write is FALSE. */
+ if (dbf->fast_write == FALSE)
__fsync (dbf);
}
@@ -85,8 +85,7 @@ _gdbm_end_update (GDBM_FILE dbf)
if (num_bytes != dbf->header->dir_size)
_gdbm_fatal (dbf, "write error");
dbf->directory_changed = FALSE;
- if (!dbf->header_changed &&
- (dbf->fast_write == FALSE || dbf->mapped_region != NULL))
+ if (!dbf->header_changed && dbf->fast_write == FALSE)
__fsync (dbf);
}
diff --git a/tests/gtload.c b/tests/gtload.c
index 550833f..b89ae1f 100644
--- a/tests/gtload.c
+++ b/tests/gtload.c
@@ -18,6 +18,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <errno.h>
#include "gdbm.h"
#include "progname.h"
@@ -37,6 +38,7 @@ main (int argc, char **argv)
GDBM_FILE dbf;
int delim = '\t';
int data_z = 0;
+ size_t mapped_size_max = 0;
while (--argc)
{
@@ -44,7 +46,7 @@ main (int argc, char **argv)
if (strcmp (arg, "-h") == 0)
{
- printf ("usage: %s [-replace] [-clear] [-blocksize=N] [-null] [-nolock] [-nommap] [-sync] [-delim=CHR] DBFILE\n", progname);
+ printf ("usage: %s [-replace] [-clear] [-blocksize=N] [-null] [-nolock] [-nommap] [-maxmap=N] [-sync] [-delim=CHR] DBFILE\n", progname);
exit (0);
}
else if (strcmp (arg, "-replace") == 0)
@@ -61,6 +63,26 @@ main (int argc, char **argv)
flags |= GDBM_SYNC;
else if (strncmp (arg, "-blocksize=", 11) == 0)
block_size = atoi (arg + 11);
+ else if (strncmp (arg, "-maxmap=", 8) == 0)
+ {
+ char *p;
+
+ errno = 0;
+ mapped_size_max = strtoul (arg + 8, &p, 10);
+
+ if (errno)
+ {
+ fprintf (stderr, "%s: ", progname);
+ perror ("maxmap");
+ exit (1);
+ }
+
+ if (*p)
+ {
+ fprintf (stderr, "%s: bad maxmap\n", progname);
+ exit (1);
+ }
+ }
else if (strncmp (arg, "-delim=", 7) == 0)
delim = arg[7];
else if (strcmp (arg, "--") == 0)
@@ -92,6 +114,17 @@ main (int argc, char **argv)
exit (1);
}
+ if (mapped_size_max)
+ {
+ if (gdbm_setopt (dbf, GDBM_SETMAXMAPSIZE, &mapped_size_max,
+ sizeof (mapped_size_max)))
+ {
+ fprintf (stderr, "gdbm_setopt failed: %s\n",
+ gdbm_strerror (gdbm_errno));
+ exit (1);
+ }
+ }
+
while (fgets (buf, sizeof buf, stdin))
{
size_t i, j;

Return to:

Send suggestions and report system problems to the System administrator.